From eaf6d6c9384ce31d025ea5b82013de7064b0c047 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 25 Jul 2016 15:52:16 -0700 Subject: Basic L2/L3 filter for rules engine (not integrated yet) and some cleanup. --- node/Filter.cpp | 229 +++++++++++++++++++++++++++++++++++++++++++++++++ node/Filter.hpp | 73 ++++++++++++++++ node/NetworkConfig.cpp | 20 ----- 3 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 node/Filter.cpp create mode 100644 node/Filter.hpp (limited to 'node') diff --git a/node/Filter.cpp b/node/Filter.cpp new file mode 100644 index 00000000..d0fccb9d --- /dev/null +++ b/node/Filter.cpp @@ -0,0 +1,229 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include + +#include "Constants.hpp" +#include "RuntimeEnvironment.hpp" +#include "Address.hpp" +#include "MAC.hpp" +#include "InetAddress.hpp" +#include "Filter.hpp" + +// Returns true if packet appears valid; pos and proto will be set +static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) +{ + if (frameLen < 40) + return false; + pos = 40; + proto = frameData[6]; + while (pos <= frameLen) { + switch(proto) { + case 0: // hop-by-hop options + case 43: // routing + case 60: // destination options + case 135: // mobility options + if ((pos + 8) > frameLen) + return false; // invalid! + proto = frameData[pos]; + pos += ((unsigned int)frameData[pos + 1] * 8) + 8; + break; + + //case 44: // fragment -- we currently can't parse these and they are deprecated in IPv6 anyway + //case 50: + //case 51: // IPSec ESP and AH -- we have to stop here since this is encrypted stuff + default: + return true; + } + } + return false; // overflow == invalid +} + +namespace ZeroTier { + +const ZT_VirtualNetworkRule *Filter::run( + const RuntimeEnvironment *RR, + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId, + const ZT_VirtualNetworkRule *rules, + const unsigned int ruleCount) +{ + uint8_t thisSetMatches = 1; + for(unsigned int rn=0;rn= 20)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); + thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto); + } else { + thisRuleMatches = 0; + } + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + const unsigned int headerLen = 4 * (frameData[0] & 0xf); + int p = -1; + switch(frameData[9]) { // IP protocol number + // All these start with 16-bit source and destination port in that order + case 0x06: // TCP + case 0x11: // UDP + case 0x84: // SCTP + case 0x88: // UDPLite + if (frameLen > (headerLen + 4)) { + unsigned int pos = headerLen + (((unsigned int)(rt == ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE)) << 1); // headerLen or +2 for destination port + p = (int)frameData[pos++] << 8; + p |= (int)frameData[pos]; + } + break; + } + thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + int p = -1; + switch(proto) { // IP protocol number + // All these start with 16-bit source and destination port in that order + case 0x06: // TCP + case 0x11: // UDP + case 0x84: // SCTP + case 0x88: // UDPLite + if (frameLen > (pos + 4)) { + p = (int)frameData[pos++] << 8; + p |= (int)frameData[pos]; + } + break; + } + thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + } else { + thisRuleMatches = 0; + } + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + /* + if (etherType == ZT_ETHERTYPE_IPV4) { + } else if (etherType == ZT_ETHERTYPE_IPV6) { + } else { + thisRuleMatches = 0; + } + */ + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); + break; + } + + // thisSetMatches remains true if the current rule matches... or does NOT match if not bit (0x80) is 1 + thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 8) & 1)); + } + + return (const ZT_VirtualNetworkRule *)0; // no matches +} + +} // namespace ZeroTier diff --git a/node/Filter.hpp b/node/Filter.hpp new file mode 100644 index 00000000..c391e958 --- /dev/null +++ b/node/Filter.hpp @@ -0,0 +1,73 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_FILTER_HPP +#define ZT_FILTER_HPP + +#include + +#include "Constants.hpp" +#include "../include/ZeroTierOne.h" + +namespace ZeroTier { + +class Address; +class RuntimeEnvironment; +class MAC; + +/** + * Network packet filter for rules engine + */ +class Filter +{ +public: + /** + * Apply a list of rules to a packet + * + * This returns the matching TARGET rule entry if there is a match or NULL + * if no match is found. + * + * @param ztSource Source ZeroTier address + * @param ztDest Destination ZeroTier address + * @param macSource Ethernet layer source address + * @param macDest Ethernet layer destination address + * @param frameData Ethernet frame data + * @param frameLen Ethernet frame payload length + * @param etherType 16-bit ethernet type ID + * @param vlanId 16-bit VLAN ID + * @param rules Pointer to array of rules + * @param ruleCount Number of rules + * @return Pointer to rules[] to matching TARGET, or NULL if no match + */ + static const ZT_VirtualNetworkRule *run( + const RuntimeEnvironment *RR, + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId, + const ZT_VirtualNetworkRule *rules, + const unsigned int ruleCount); +}; + +} // namespace ZeroTier + +#endif diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 9d5c5f17..3a307fe7 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -245,17 +245,6 @@ bool NetworkConfig::toDictionary(Dictionary &d,b tmp.append((uint16_t)rules[i].v.frameSize[0]); tmp.append((uint16_t)rules[i].v.frameSize[1]); break; - case ZT_NETWORK_RULE_MATCH_TCP_RELATIVE_SEQUENCE_NUMBER_RANGE: - tmp.append((uint8_t)8); - tmp.append((uint32_t)rules[i].v.tcpseq[0]); - tmp.append((uint32_t)rules[i].v.tcpseq[1]); - break; - case ZT_NETWORK_RULE_MATCH_COM_FIELD_GE: - case ZT_NETWORK_RULE_MATCH_COM_FIELD_LE: - tmp.append((uint8_t)16); - tmp.append((uint64_t)rules[i].v.comIV[0]); - tmp.append((uint64_t)rules[i].v.comIV[1]); - break; } } if (tmp.size()) { @@ -472,15 +461,6 @@ bool NetworkConfig::fromDictionary(const Dictionary(p); rules[ruleCount].v.frameSize[0] = tmp.at(p + 2); break; - case ZT_NETWORK_RULE_MATCH_TCP_RELATIVE_SEQUENCE_NUMBER_RANGE: - rules[ruleCount].v.tcpseq[0] = tmp.at(p); - rules[ruleCount].v.tcpseq[1] = tmp.at(p + 4); - break; - case ZT_NETWORK_RULE_MATCH_COM_FIELD_GE: - case ZT_NETWORK_RULE_MATCH_COM_FIELD_LE: - rules[ruleCount].v.comIV[0] = tmp.at(p); - rules[ruleCount].v.comIV[1] = tmp.at(p + 8); - break; } p += fieldLen; ++ruleCount; -- cgit v1.2.3 From 7404eb46c4279b1e2ecce29aece14e15fbedbffd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 25 Jul 2016 16:51:10 -0700 Subject: Integration of Filter into inbound and outbound packet path. --- node/Filter.cpp | 54 ++++++++++++++++++++++++--------- node/Filter.hpp | 10 +++--- node/IncomingPacket.cpp | 81 +++++++++++++++++++++++++++++++++++++++---------- node/NetworkConfig.hpp | 20 ------------ node/Switch.cpp | 44 +++++++++++++++++++++++---- version.h | 2 +- 6 files changed, 150 insertions(+), 61 deletions(-) (limited to 'node') diff --git a/node/Filter.cpp b/node/Filter.cpp index d0fccb9d..1510f820 100644 --- a/node/Filter.cpp +++ b/node/Filter.cpp @@ -24,6 +24,9 @@ #include "MAC.hpp" #include "InetAddress.hpp" #include "Filter.hpp" +#include "Packet.hpp" +#include "Switch.hpp" +#include "Topology.hpp" // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) @@ -56,8 +59,9 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig namespace ZeroTier { -const ZT_VirtualNetworkRule *Filter::run( +bool Filter::run( const RuntimeEnvironment *RR, + const uint64_t nwid, const Address &ztSource, const Address &ztDest, const MAC &macSource, @@ -69,21 +73,49 @@ const ZT_VirtualNetworkRule *Filter::run( const ZT_VirtualNetworkRule *rules, const unsigned int ruleCount) { + // For each set of rules we start by assuming that they match (since no constraints + // yields a 'match all' rule). uint8_t thisSetMatches = 1; + for(unsigned int rn=0;rnidentity.address(),Packet::VERB_EXT_FRAME); + outp.append(nwid); + outp.append((unsigned char)0x00); // TODO: should maybe include COM if needed + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,frameLen); + outp.compress(); + RR->sw->send(outp,true,nwid); + } + // For REDIRECT we will want to DROP at this node. For TEE we ACCEPT at this node but + // also forward it along as we just did. + return (rt != ZT_NETWORK_RULE_ACTION_REDIRECT); + } + } else { + // Otherwise start a new set, assuming that it will match + thisSetMatches = 1; + } break; + // A rule can consist of one or more MATCH criterion case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); break; @@ -206,24 +238,18 @@ const ZT_VirtualNetworkRule *Filter::run( } break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - /* - if (etherType == ZT_ETHERTYPE_IPV4) { - } else if (etherType == ZT_ETHERTYPE_IPV6) { - } else { - thisRuleMatches = 0; - } - */ + // TODO: not supported yet break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; } - // thisSetMatches remains true if the current rule matches... or does NOT match if not bit (0x80) is 1 - thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 8) & 1)); + // thisSetMatches remains true if the current rule matched... or does NOT match if not bit (0x80) is 1 + thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); } - return (const ZT_VirtualNetworkRule *)0; // no matches + return false; // no matches, no rules, default action is therefore DROP } } // namespace ZeroTier diff --git a/node/Filter.hpp b/node/Filter.hpp index c391e958..f8b66134 100644 --- a/node/Filter.hpp +++ b/node/Filter.hpp @@ -39,9 +39,11 @@ public: /** * Apply a list of rules to a packet * - * This returns the matching TARGET rule entry if there is a match or NULL - * if no match is found. + * This returns whether or not the packet should be accepted and may also + * take other actions for e.g. the TEE and REDIRECT targets. * + * @param RR ZeroTier runtime environment (context) + * @param nwid ZeroTier network ID * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -52,10 +54,10 @@ public: * @param vlanId 16-bit VLAN ID * @param rules Pointer to array of rules * @param ruleCount Number of rules - * @return Pointer to rules[] to matching TARGET, or NULL if no match */ - static const ZT_VirtualNetworkRule *run( + static bool run( const RuntimeEnvironment *RR, + const uint64_t nwid, const Address &ztSource, const Address &ztDest, const MAC &macSource, diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 37af8425..b666e42c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -37,6 +37,7 @@ #include "Cluster.hpp" #include "Node.hpp" #include "DeferredPackets.hpp" +#include "Filter.hpp" namespace ZeroTier { @@ -550,13 +551,27 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr } const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); - if (!network->config().permitsEtherType(etherType)) { - TRACE("dropped FRAME from %s(%s): ethertype %.4x not allowed on %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); - return true; + const MAC sourceMac(peer->address(),network->id()); + const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; + const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; + if (Filter::run( + RR, + network->id(), + peer->address(), + RR->identity.address(), + sourceMac, + network->mac(), + frameData, + frameLen, + etherType, + 0, + network->config().rules, + network->config().ruleCount)) + { + RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); + } else { + TRACE("dropped FRAME from %s(%s): Filter::run() == false (will still log packet as received)",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); } - - const unsigned int payloadLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - RR->node->putFrame(network->id(),network->userPtr(),MAC(peer->address(),network->id()),network->mac(),etherType,0,field(ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,payloadLen),payloadLen); } peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP); @@ -594,10 +609,6 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE); - if (!network->config().permitsEtherType(etherType)) { - TRACE("dropped EXT_FRAME from %s(%s): ethertype %.4x not allowed on network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); - return true; - } const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO); const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM); @@ -626,8 +637,26 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,payloadLen),payloadLen); + const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); + const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); + if (Filter::run( + RR, + network->id(), + peer->address(), + RR->identity.address(), + from, + to, + frameData, + frameLen, + etherType, + 0, + network->config().rules, + network->config().ruleCount)) + { + RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); + } else { + TRACE("dropped EXT_FRAME from %s(%s): Filter::run() == false (will still log packet as received)",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); + } } peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP); @@ -870,11 +899,11 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share const MulticastGroup to(MAC(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI)); const unsigned int etherType = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE); - const unsigned int payloadLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); + const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); - //TRACE("<address().toString().c_str(),flags,payloadLen); + //TRACE("<address().toString().c_str(),flags,frameLen); - if ((payloadLen > 0)&&(payloadLen <= ZT_IF_MTU)) { + if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); return true; @@ -893,7 +922,27 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share } } - RR->node->putFrame(network->id(),network->userPtr(),from,to.mac(),etherType,0,field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,payloadLen),payloadLen); + const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); + if (Filter::run( + RR, + network->id(), + peer->address(), + RR->identity.address(), + from, + to.mac(), + frameData, + frameLen, + etherType, + 0, + network->config().rules, + network->config().ruleCount)) + { + RR->node->putFrame(network->id(),network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); + } else { + TRACE("dropped MULTICAST_FRAME from %s(%s): Filter::run() == false (will still do implicit gather)",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); + // Note: we continue here since we still do implicit gather in this case... we just do not putFrame() if it + // fails the filter check. + } } if (gatherLimit) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 5271c5ad..f2dab6d3 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -215,26 +215,6 @@ public: return *this; } - /** - * @param etherType Ethernet frame type to check - * @return True if allowed on this network - */ - inline bool permitsEtherType(unsigned int etherType) const - { - unsigned int et = 0; - for(unsigned int i=0;i &network,const MAC &from,c if (to == network->mac()) return; - // Check to make sure this protocol is allowed on this network - if (!network->config().permitsEtherType(etherType)) { - TRACE("%.16llx: ignored tap: %s -> %s: ethertype %s not allowed on network %.16llx",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),(unsigned long long)network->id()); - return; - } - // Check if this packet is from someone other than the tap -- i.e. bridged in bool fromBridged = false; if (from != network->mac()) { @@ -443,6 +438,24 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),len); + if (!Filter::run( + RR, + network->id(), + RR->identity.address(), + Address(), // 0 destination ZT address for multicasts since this is unknown at time of send + from, + to, + (const uint8_t *)data, + len, + etherType, + vlanId, + network->config().rules, + network->config().ruleCount)) + { + TRACE("%.16llx: %s -> %s %s packet not sent: Filter::run() == false (multicast)",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + return; + } + RR->mc->send( ((!network->config().isPublic())&&(network->config().com)) ? &(network->config().com) : (const CertificateOfMembership *)0, network->config().multicastLimit, @@ -463,6 +476,25 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this SharedPtr toPeer(RR->topology->getPeer(toZT)); + + if (!Filter::run( + RR, + network->id(), + RR->identity.address(), + toZT, + from, + to, + (const uint8_t *)data, + len, + etherType, + vlanId, + network->config().rules, + network->config().ruleCount)) + { + TRACE("%.16llx: %s -> %s %s packet not sent: Filter::run() == false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + return; + } + const bool includeCom = ( (network->config().isPrivate()) && (network->config().com) && ((!toPeer)||(toPeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) ); if ((fromBridged)||(includeCom)) { Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME); diff --git a/version.h b/version.h index 18300110..737b3783 100644 --- a/version.h +++ b/version.h @@ -32,6 +32,6 @@ /** * Revision */ -#define ZEROTIER_ONE_VERSION_REVISION 14 +#define ZEROTIER_ONE_VERSION_REVISION 15 #endif -- cgit v1.2.3 From 088bbd1c089b443f1ed1a84089f666833b97399d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 25 Jul 2016 17:03:26 -0700 Subject: Filter fixes. --- node/Filter.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Filter.cpp b/node/Filter.cpp index 1510f820..a4de7201 100644 --- a/node/Filter.cpp +++ b/node/Filter.cpp @@ -27,6 +27,7 @@ #include "Packet.hpp" #include "Switch.hpp" #include "Topology.hpp" +#include "Node.hpp" // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) @@ -89,10 +90,7 @@ bool Filter::run( case ZT_NETWORK_RULE_ACTION_REDIRECT: if (thisSetMatches) { // This set did match, so perform action! - if (rt == ZT_NETWORK_RULE_ACTION_DROP) { - // DROP means do nothing at all. - return false; - } else { + if (rt != ZT_NETWORK_RULE_ACTION_DROP) { if ((rt == ZT_NETWORK_RULE_ACTION_TEE)||(rt == ZT_NETWORK_RULE_ACTION_REDIRECT)) { // Tee and redirect both want this frame copied to somewhere else. Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); @@ -109,11 +107,13 @@ bool Filter::run( // also forward it along as we just did. return (rt != ZT_NETWORK_RULE_ACTION_REDIRECT); } + return false; } else { // Otherwise start a new set, assuming that it will match + //TRACE("[%u] %u previous set did not match, starting next",rn,(unsigned int)rt); thisSetMatches = 1; } - break; + continue; // A rule can consist of one or more MATCH criterion case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: @@ -247,6 +247,8 @@ bool Filter::run( // thisSetMatches remains true if the current rule matched... or does NOT match if not bit (0x80) is 1 thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); + + //TRACE("[%u] %u result==%u set==%u",rn,(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); } return false; // no matches, no rules, default action is therefore DROP -- cgit v1.2.3 From 4929be08f77dbdc2c0277dc99c2b5438ede4b137 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 26 Jul 2016 12:33:51 -0700 Subject: Cleanup and stub out new object transfer messages. --- node/IncomingPacket.cpp | 32 ++++++---------- node/IncomingPacket.hpp | 3 +- node/Packet.cpp | 3 +- node/Packet.hpp | 98 +++++++++++++++++++++++++++++++------------------ 4 files changed, 77 insertions(+), 59 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b666e42c..e52b3f91 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -108,7 +108,6 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,bool deferred) case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); case Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return _doNETWORK_MEMBERSHIP_CERTIFICATE(RR,peer); case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); - case Packet::VERB_NETWORK_CONFIG_REFRESH: return _doNETWORK_CONFIG_REFRESH(RR,peer); case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); @@ -162,8 +161,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { /* Note: certificates are public so it's safe to push them to anyone - * who asks. We won't communicate unless we also get a certificate - * from the remote that agrees. */ + * who asks. */ SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->hasConfig())&&(network->config().com)) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE); @@ -805,24 +803,6 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons return true; } -bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer) -{ - try { - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD; - while ((ptr + 8) <= size()) { - uint64_t nwid = at(ptr); - SharedPtr nw(RR->node->network(nwid)); - if ((nw)&&(peer->address() == nw->controller())) - nw->requestConfiguration(); - ptr += 8; - } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP); - } catch ( ... ) { - TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); - } - return true; -} - bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { @@ -1320,6 +1300,16 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const return true; } +bool IncomingPacket::_doREQUEST_OBJECT(const RuntimeEnvironment *RR,const SharedPtr &peer) +{ + return true; +} + +bool IncomingPacket::_doOBJECT_UPDATED(const RuntimeEnvironment *RR,const SharedPtr &peer) +{ + return true; +} + void IncomingPacket::computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16]) { unsigned char salsabuf[131072]; // 131072 == protocol constant, size of memory buffer for this proof of work function diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index cd0b7dcf..ab7afd51 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -174,13 +174,14 @@ private: bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr &peer); + bool _doREQUEST_OBJECT(const RuntimeEnvironment *RR,const SharedPtr &peer); + bool _doOBJECT_UPDATED(const RuntimeEnvironment *RR,const SharedPtr &peer); // Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to communicate void _sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr &peer,uint64_t nwid); diff --git a/node/Packet.cpp b/node/Packet.cpp index 3330a927..5152f572 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -40,13 +40,14 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; case VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return "NETWORK_MEMBERSHIP_CERTIFICATE"; case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; - case VERB_NETWORK_CONFIG_REFRESH: return "NETWORK_CONFIG_REFRESH"; case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST"; case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT"; case VERB_REQUEST_PROOF_OF_WORK: return "REQUEST_PROOF_OF_WORK"; + case VERB_REQUEST_OBJECT: return "REQUEST_OBJECT"; + case VERB_OBJECT_UPDATED: return "OBJECT_UPDATED"; } return "(unknown)"; } diff --git a/node/Packet.hpp b/node/Packet.hpp index 3d95b0ba..211c3aa5 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -715,53 +715,23 @@ public: VERB_NETWORK_MEMBERSHIP_CERTIFICATE = 10, /** - * Network configuration request: - * <[8] 64-bit network ID> - * <[2] 16-bit length of request meta-data dictionary> - * <[...] string-serialized request meta-data> - * [<[8] 64-bit revision of netconf we currently have>] + * DEPRECATED but still supported, interpreted as an object request: + * + * /controller/network//member/ * - * This message requests network configuration from a node capable of - * providing it. If the optional revision is included, a response is - * only generated if there is a newer network configuration available. + * When received in this manner the response is sent via the old + * OK(NETWORK_CONFIG_REQUEST) instead of OK(REQUEST_OBJECT). * * OK response payload: * <[8] 64-bit network ID> * <[2] 16-bit length of network configuration dictionary> * <[...] network configuration dictionary> * - * OK returns a Dictionary (string serialized) containing the network's - * configuration and IP address assignment information for the querying - * node. It also contains a membership certificate that the querying - * node can push to other peers to demonstrate its right to speak on - * a given network. - * - * When a new network configuration is received, another config request - * should be sent with the new netconf's revision. This confirms receipt - * and also causes any subsequent changes to rapidly propagate as this - * cycle will repeat until there are no changes. This is optional but - * recommended behavior. - * * ERROR response payload: * <[8] 64-bit network ID> - * - * UNSUPPORTED_OPERATION is returned if this service is not supported, - * and OBJ_NOT_FOUND if the queried network ID was not found. */ VERB_NETWORK_CONFIG_REQUEST = 11, - /** - * Network configuration refresh request: - * <[...] array of 64-bit network IDs> - * - * This can be sent by the network controller to inform a node that it - * should now make a NETWORK_CONFIG_REQUEST. - * - * It does not generate an OK or ERROR message, and is treated only as - * a hint to refresh now. - */ - VERB_NETWORK_CONFIG_REFRESH = 12, - /** * Request endpoints for multicast distribution: * <[8] 64-bit network ID> @@ -1030,7 +1000,63 @@ public: * * ERROR has no payload. */ - VERB_REQUEST_PROOF_OF_WORK = 19 + VERB_REQUEST_PROOF_OF_WORK = 19, + + /** + * Request an object or a chunk of an object with optional meta-data: + * <[8] 64-bit chunk offset> + * <[2] 16-bit chunk length or 0 for any / sender-preferred> + * <[2] 16-bit object path length in bytes> + * <[...] object path> + * <[2] 16-bit length of request meta-data dictionary> + * <[...] request meta-data dictionary> + * + * This is used to request an object. Objects can be things like network + * configs, software updates, etc. This provides an in-band way to + * distribute such things and obsoletes the network config specific + * messages. (They are still supported for backward compatibility.) + * + * The use of path and request/response meta-data makes the semantics of + * this analogous to HTTP POST, and it could therefore be mapped to + * HTTP POST requests to permit plugins that leverage the ZT protocol + * to do out-of-band things like special authentication, etc. + * + * Large objects can be transferred via repeated calls with higher and + * higher chunk offsets and then SHA-512 verified on receipt, but this is + * not efficient. It should not be used heavily as an alternative to + * TCP. It's a bit more like X-Modem and other old-school SEND/ACK + * protocols. It is potentially a good idea for software updates since + * it means that ZT can update itself even on networks with no "vanilla" + * Internet access. + * + * OK and ERROR responses are optional but recommended. ERROR responses + * can include OBJECT_NOT_FOUND. + * + * OK response payload: + * <[16] first 16 bytes of SHA-512 of complete object> + * <[8] 64-bit total object size> + * <[8] 64-bit chunk offset> + * <[2] 16-bit length of chunk payload> + * <[...] chunk payload> + */ + VERB_REQUEST_OBJECT = 20, + + /** + * Notification of a remote object update: + * <[8] 64-bit total object size or 0 if unspecified here> + * <[16] first 16 bytes of SHA-512 of object (if size specified)> + * <[2] 16-bit length of object path> + * <[...] object path> + * <[2] 16-bit length of meta-data dictionary> + * <[...] meta-data dictionary> + * + * This can be sent to notify another peer that an object has updated and + * should be re-requested. The receiving peer is not required to do anything + * or send anything in response to this. If the first size field is zero, the + * SHA-512 hash is also unspecified and should be zero. This means that the + * object was updated but must be re-requested. + */ + VERB_OBJECT_UPDATED = 21 }; /** -- cgit v1.2.3 From 22e44c762bf77aefe988ed7b6874054f84f95b75 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 28 Jul 2016 10:58:10 -0700 Subject: More rules engine work: key/value pair matching for microsegmentation. --- controller/SqliteNetworkController.cpp | 40 +++++++++++++++++--- controller/SqliteNetworkController.hpp | 1 - controller/schema.sql | 27 +++++--------- controller/schema.sql.c | 27 +++++--------- include/ZeroTierOne.h | 11 ++++-- node/CertificateOfMembership.hpp | 67 ++++++++++++---------------------- osdep/Thread.hpp | 11 ++++-- 7 files changed, 92 insertions(+), 92 deletions(-) (limited to 'node') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 65051744..81017897 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -66,8 +66,8 @@ // Stored in database as schemaVersion key in Config. // If not present, database is assumed to be empty and at the current schema version // and this key/value is added automatically. -#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 4 -#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "4" +#define ZT_NETCONF_SQLITE_SCHEMA_VERSION 5 +#define ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR "5" // API version reported via JSON control plane #define ZT_NETCONF_CONTROLLER_API_VERSION 2 @@ -334,6 +334,38 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c } } + if (schemaVersion < 5) { + // Upgrade old rough draft Rule table to new release format + if (sqlite3_exec(_db, + "DROP INDEX Rule_networkId_ruleNo;\n" + "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" + "CREATE TABLE Rule (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " policyId varchar(32),\n" + " ruleNo integer NOT NULL,\n" + " ruleType integer NOT NULL DEFAULT(0),\n" + " \"addr\" blob(16),\n" + " \"int1\" integer,\n" + " \"int2\" integer,\n" + " \"int3\" integer,\n" + " \"int4\" integer\n" + ");\n" + "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" + "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" + "DROP TABLE RuleOld;\n" + "CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n" + "CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId);\n" + "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 5; + } + } + if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { sqlite3_close(_db); throw std::runtime_error("SqliteNetworkController database schema version mismatch"); @@ -365,8 +397,7 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) /* Rule */ - ||(sqlite3_prepare_v2(_db,"SELECT etherType FROM Rule WHERE networkId = ? AND \"action\" = 'accept'",-1,&_sGetEtherTypesFromRuleTable,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcP,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,flags,invFlags,\"action\") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) @@ -457,7 +488,6 @@ SqliteNetworkController::~SqliteNetworkController() sqlite3_finalize(_sCreateMember); sqlite3_finalize(_sGetNodeIdentity); sqlite3_finalize(_sCreateOrReplaceNode); - sqlite3_finalize(_sGetEtherTypesFromRuleTable); sqlite3_finalize(_sGetActiveBridges); sqlite3_finalize(_sGetIpAssignmentsForNode); sqlite3_finalize(_sGetIpAssignmentPools); diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp index 145788c7..b917f6fb 100644 --- a/controller/SqliteNetworkController.hpp +++ b/controller/SqliteNetworkController.hpp @@ -137,7 +137,6 @@ private: sqlite3_stmt *_sCreateMember; sqlite3_stmt *_sGetNodeIdentity; sqlite3_stmt *_sCreateOrReplaceNode; - sqlite3_stmt *_sGetEtherTypesFromRuleTable; sqlite3_stmt *_sGetActiveBridges; sqlite3_stmt *_sGetIpAssignmentsForNode; sqlite3_stmt *_sGetIpAssignmentPools; diff --git a/controller/schema.sql b/controller/schema.sql index 105db924..479daa68 100644 --- a/controller/schema.sql +++ b/controller/schema.sql @@ -96,24 +96,15 @@ CREATE UNIQUE INDEX Relay_networkId_address ON Relay (networkId,address); CREATE TABLE Rule ( networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE, + policyId varchar(32), ruleNo integer NOT NULL, - nodeId char(10) REFERENCES Node(id), - sourcePort char(10), - destPort char(10), - vlanId integer, - vlanPcp integer, - etherType integer, - macSource char(12), - macDest char(12), - ipSource varchar(64), - ipDest varchar(64), - ipTos integer, - ipProtocol integer, - ipSourcePort integer, - ipDestPort integer, - flags integer, - invFlags integer, - "action" varchar(4096) NOT NULL DEFAULT('accept') + ruleType integer NOT NULL DEFAULT(0), + "addr" blob(16), + "int1" integer, + "int2" integer, + "int3" integer, + "int4" integer ); -CREATE UNIQUE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo); +CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo); +CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId); diff --git a/controller/schema.sql.c b/controller/schema.sql.c index dab34138..e84ee766 100644 --- a/controller/schema.sql.c +++ b/controller/schema.sql.c @@ -97,25 +97,16 @@ "\n"\ "CREATE TABLE Rule (\n"\ " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n"\ +" policyId varchar(32),\n"\ " ruleNo integer NOT NULL,\n"\ -" nodeId char(10) REFERENCES Node(id),\n"\ -" sourcePort char(10),\n"\ -" destPort char(10),\n"\ -" vlanId integer,\n"\ -" vlanPcp integer,\n"\ -" etherType integer,\n"\ -" macSource char(12),\n"\ -" macDest char(12),\n"\ -" ipSource varchar(64),\n"\ -" ipDest varchar(64),\n"\ -" ipTos integer,\n"\ -" ipProtocol integer,\n"\ -" ipSourcePort integer,\n"\ -" ipDestPort integer,\n"\ -" flags integer,\n"\ -" invFlags integer,\n"\ -" \"action\" varchar(4096) NOT NULL DEFAULT('accept')\n"\ +" ruleType integer NOT NULL DEFAULT(0),\n"\ +" \"addr\" blob(16),\n"\ +" \"int1\" integer,\n"\ +" \"int2\" integer,\n"\ +" \"int3\" integer,\n"\ +" \"int4\" integer\n"\ ");\n"\ "\n"\ -"CREATE UNIQUE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n"\ +"CREATE INDEX Rule_networkId_ruleNo ON Rule (networkId, ruleNo);\n"\ +"CREATE INDEX Rule_networkId_policyId ON Rule (networkId, policyId);\n"\ "" diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 0d1ddd4b..6abc04f2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -391,12 +391,15 @@ enum ZT_VirtualNetworkType /** * The type of a virtual network rules table entry * - * These must range from 0 to 127 (0x7f). + * These must range from 0 to 127 (0x7f) because the most significant bit + * is reserved as a NOT flag. * * Each rule is composed of one or more MATCHes followed by an ACTION. */ enum ZT_VirtualNetworkRuleType { + // 0 to 31 reserved for actions + /** * Drop frame */ @@ -408,16 +411,16 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_ACTION_ACCEPT = 1, /** - * Forward a copy of this frame to an observer + * Forward a copy of this frame to an observer (by ZT address) */ ZT_NETWORK_RULE_ACTION_TEE = 2, /** - * Explicitly redirect this frame to another device (ignored if this is the target device) + * Drop and redirect this frame to another node (by ZT address) */ ZT_NETWORK_RULE_ACTION_REDIRECT = 3, - // <32 == actions + // 32 to 127 reserved for match criteria /** * Source ZeroTier address -- analogous to an Ethernet port ID on a switch diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 0342bc33..9f251f6b 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -44,9 +44,9 @@ #define ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA (ZT_NETWORK_AUTOCONF_DELAY * 5) /** - * Maximum number of qualifiers in a COM + * Maximum number of qualifiers allowed in a COM (absolute max: 65535) */ -#define ZT_NETWORK_COM_MAX_QUALIFIERS 16 +#define ZT_NETWORK_COM_MAX_QUALIFIERS 256 namespace ZeroTier { @@ -87,14 +87,15 @@ public: */ enum Type { - COM_UINT64_ED25519 = 1 // tuples of unsigned 64's signed with Ed25519 + // tuples of unsigned 64's signed with Ed25519 + COM_UINT64_ED25519 = 1 }; /** * Reserved qualifier IDs * - * IDs below 65536 should be considered reserved for future global - * assignment here. + * IDs below 1024 are reserved for use as standard IDs. Others are available + * for user-defined use. * * Addition of new required fields requires that code in hasRequiredFields * be updated as well. @@ -126,12 +127,11 @@ public: }; /** - * Create an empty certificate + * Create an empty certificate of membership */ - CertificateOfMembership() : - _qualifierCount(0) + CertificateOfMembership() { - memset(_signature.data,0,_signature.size()); + memset(this,0,sizeof(CertificateOfMembership)); } CertificateOfMembership(const CertificateOfMembership &c) @@ -168,22 +168,6 @@ public: return *this; } -#ifdef ZT_SUPPORT_OLD_STYLE_NETCONF - /** - * Create from string-serialized data - * - * @param s String-serialized COM - */ - CertificateOfMembership(const char *s) { fromString(s); } - - /** - * Create from string-serialized data - * - * @param s String-serialized COM - */ - CertificateOfMembership(const std::string &s) { fromString(s.c_str()); } -#endif // ZT_SUPPORT_OLD_STYLE_NETCONF - /** * Create from binary-serialized COM in buffer * @@ -201,24 +185,6 @@ public: */ inline operator bool() const throw() { return (_qualifierCount != 0); } - /** - * Check for presence of all required fields common to all networks - * - * @return True if all required fields are present - */ - inline bool hasRequiredFields() const - { - if (_qualifierCount < 3) - return false; - if (_qualifiers[0].id != COM_RESERVED_ID_REVISION) - return false; - if (_qualifiers[1].id != COM_RESERVED_ID_NETWORK_ID) - return false; - if (_qualifiers[2].id != COM_RESERVED_ID_ISSUED_TO) - return false; - return true; - } - /** * @return Maximum delta for mandatory revision field or 0 if field missing */ @@ -279,6 +245,21 @@ public: void setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta); inline void setQualifier(ReservedId id,uint64_t value,uint64_t maxDelta) { setQualifier((uint64_t)id,value,maxDelta); } + /** + * Get the value of a qualifier field + * + * @param id Qualifier ID + * @return Value or 0 if not found + */ + inline uint64_t getQualifierValue(uint64_t id) + { + for(unsigned int i=0;i<_qualifierCount;++i) { + if (_qualifiers[i].id == id) + return _qualifiers[i].value; + } + return 0; + } + #ifdef ZT_SUPPORT_OLD_STYLE_NETCONF /** * @return String-serialized representation of this certificate diff --git a/osdep/Thread.hpp b/osdep/Thread.hpp index 4f90dc0b..9f6fb5a8 100644 --- a/osdep/Thread.hpp +++ b/osdep/Thread.hpp @@ -126,12 +126,17 @@ public: { memset(&_tid,0,sizeof(_tid)); pthread_attr_init(&_tattr); -#ifdef __LINUX__ - pthread_attr_setstacksize(&_tattr,8388608); // for MUSL libc and others, has no effect in normal glibc environments -#endif + // This corrects for systems with abnormally small defaults (musl) and also + // shrinks the stack on systems with large defaults to save a bit of memory. + pthread_attr_setstacksize(&_tattr,524288); _started = false; } + ~Thread() + { + pthread_attr_destroy(&_tattr); + } + Thread(const Thread &t) throw() { -- cgit v1.2.3 From d3b0081447940ee3cad4f39bc6e022bd7434402b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 28 Jul 2016 12:09:58 -0700 Subject: Cleanup... --- include/ZeroTierOne.h | 2 +- node/CertificateOfMembership.hpp | 33 +++------------------------------ node/NetworkConfig.cpp | 8 +++++--- 3 files changed, 9 insertions(+), 34 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 6abc04f2..b03abf86 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -565,7 +565,7 @@ typedef struct /** * Packet characteristic flags being matched */ - uint64_t characteristics; + uint64_t characteristics[2]; /** * IP port range -- start-end inclusive -- host byte order diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 9f251f6b..430ed785 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -79,18 +79,6 @@ namespace ZeroTier { class CertificateOfMembership { public: - /** - * Certificate type codes, used in serialization - * - * Only one so far, and only one hopefully there shall be for quite some - * time. - */ - enum Type - { - // tuples of unsigned 64's signed with Ed25519 - COM_UINT64_ED25519 = 1 - }; - /** * Reserved qualifier IDs * @@ -245,21 +233,6 @@ public: void setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta); inline void setQualifier(ReservedId id,uint64_t value,uint64_t maxDelta) { setQualifier((uint64_t)id,value,maxDelta); } - /** - * Get the value of a qualifier field - * - * @param id Qualifier ID - * @return Value or 0 if not found - */ - inline uint64_t getQualifierValue(uint64_t id) - { - for(unsigned int i=0;i<_qualifierCount;++i) { - if (_qualifiers[i].id == id) - return _qualifiers[i].value; - } - return 0; - } - #ifdef ZT_SUPPORT_OLD_STYLE_NETCONF /** * @return String-serialized representation of this certificate @@ -322,7 +295,7 @@ public: template inline void serialize(Buffer &b) const { - b.append((unsigned char)COM_UINT64_ED25519); + b.append((uint8_t)1); b.append((uint16_t)_qualifierCount); for(unsigned int i=0;i<_qualifierCount;++i) { b.append(_qualifiers[i].id); @@ -342,8 +315,8 @@ public: _qualifierCount = 0; _signedBy.zero(); - if (b[p++] != COM_UINT64_ED25519) - throw std::invalid_argument("invalid type"); + if (b[p++] != 1) + throw std::invalid_argument("invalid field type"); unsigned int numq = b.template at(p); p += sizeof(uint16_t); uint64_t lastId = 0; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 3a307fe7..7a7abdd6 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -237,8 +237,9 @@ bool NetworkConfig::toDictionary(Dictionary &d,b tmp.append((uint16_t)rules[i].v.port[1]); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - tmp.append((uint8_t)8); - tmp.append((uint64_t)rules[i].v.characteristics); + tmp.append((uint8_t)16); + tmp.append((uint64_t)rules[i].v.characteristics[0]); + tmp.append((uint64_t)rules[i].v.characteristics[1]); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: tmp.append((uint8_t)4); @@ -455,7 +456,8 @@ bool NetworkConfig::fromDictionary(const Dictionary(p + 2); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - rules[ruleCount].v.characteristics = tmp.at(p); + rules[ruleCount].v.characteristics[0] = tmp.at(p); + rules[ruleCount].v.characteristics[1] = tmp.at(p + 8); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: rules[ruleCount].v.frameSize[0] = tmp.at(p); -- cgit v1.2.3 From ecc1324bb0b2435d958947148984a2bd1f630ed7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 2 Aug 2016 13:36:17 -0700 Subject: Rules engine work: capability based security model with tags and capabilities, and some cleanup across other places. --- include/ZeroTierOne.h | 45 ++++- node/Capability.hpp | 389 +++++++++++++++++++++++++++++++++++++++ node/CertificateOfMembership.hpp | 2 +- node/Identity.cpp | 2 +- node/Identity.hpp | 17 +- node/Packet.hpp | 14 +- node/Tag.hpp | 171 +++++++++++++++++ node/World.hpp | 12 +- 8 files changed, 614 insertions(+), 38 deletions(-) create mode 100644 node/Capability.hpp create mode 100644 node/Tag.hpp (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index b03abf86..c4696e7d 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -102,14 +102,14 @@ extern "C" { #define ZT_MAX_NETWORK_PINNED 16 /** - * Maximum number of rule table entries per network (can be increased) + * Maximum number of multicast group subscriptions per network */ -#define ZT_MAX_NETWORK_RULES 256 +#define ZT_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS 4096 /** - * Maximum number of multicast group subscriptions per network + * Maximum number of base (non-capability) network rules */ -#define ZT_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS 4096 +#define ZT_MAX_NETWORK_RULES 256 /** * Maximum number of direct network paths to a given peer @@ -121,6 +121,21 @@ extern "C" { */ #define ZT_MAX_TRUSTED_PATHS 16 +/** + * Maximum number of rules per capability + */ +#define ZT_MAX_CAPABILITY_RULES 64 + +/** + * Maximum length of a capbility's short descriptive name + */ +#define ZT_MAX_CAPABILITY_NAME_LENGTH 63 + +/** + * Global maximum length for capability chain of custody (including initial issue) + */ +#define ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH 7 + /** * Maximum number of hops in a ZeroTier circuit test * @@ -516,9 +531,6 @@ enum ZT_VirtualNetworkRuleType /** * Network flow rule * - * NOTE: Currently (1.1.x) only etherType is supported! Other things will - * have no effect until the rules engine is fully implemented. - * * Rules are stored in a table in which one or more match entries is followed * by an action. If more than one match precedes an action, the rule is * the AND of all matches. An action with no match is always taken since it @@ -619,6 +631,25 @@ typedef struct } v; } ZT_VirtualNetworkRule; +typedef struct +{ + /** + * 128-bit ID (GUID) of this capability + */ + uint64_t id[2]; + + /** + * Expiration time (measured vs. network config timestamp issued by controller) + */ + uint64_t expiration; + + + struct { + uint64_t from; + uint64_t to; + } custody[ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH]; +} ZT_VirtualNetworkCapability; + /** * A route to be pushed on a virtual network */ diff --git a/node/Capability.hpp b/node/Capability.hpp new file mode 100644 index 00000000..6de4e0a1 --- /dev/null +++ b/node/Capability.hpp @@ -0,0 +1,389 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CAPABILITY_HPP +#define ZT_CAPABILITY_HPP + +#include +#include +#include + +#include "Constants.hpp" +#include "Address.hpp" +#include "C25519.hpp" +#include "Utils.hpp" +#include "Buffer.hpp" +#include "Identity.hpp" +#include "../include/ZeroTierOne.h" + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * A set of grouped and signed network flow rules + * + * The use of capabilities implements capability-based security on ZeroTIer + * virtual networks for efficient and manageable network micro-segmentation. + * + * On the sending side the sender does the following for each packet: + * + * (1) Evaluates its capabilities in ascending order of ID to determine + * which capability allows it to transmit this packet. + * (2) If it has not done so lately, it then sends this capability to the + * receving peer ("presents" it). + * (3) The sender then sends the packet. + * + * On the receiving side the receiver does the following for each packet: + * + * (1) Evaluates the capabilities of the sender (that the sender has + * presented) to determine if the sender was allowed to send this. + * (2) Evaluates its own capabilities to determine if it should receive + * and process this packet. + * (3) If both check out, it receives the packet. + * + * Note that rules in capabilities can do other things as well such as TEE + * or REDIRECT packets. See Filter and ZT_VirtualNetworkRule. + */ +class Capability +{ +public: + Capability() + { + memset(this,0,sizeof(Capability)); + } + + /** + * @param id Capability ID + * @param nwid Network ID + * @param expiration Expiration relative to network config timestamp + * @param name Capability short name (max strlen == ZT_MAX_CAPABILITY_NAME_LENGTH, overflow ignored) + * @param mccl Maximum custody chain length (1 to create non-transferrable capability) + * @param rules Network flow rules for this capability + * @param ruleCount Number of flow rules + */ + Capability(uint32_t id,uint64_t nwid,uint64_t expiration,const char *name,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + { + memset(this,0,sizeof(Capability)); + _nwid = nwid; + _expiration = expiration; + _id = id; + _maxCustodyChainLength = (mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1; + _ruleCount = (ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES; + if (_ruleCount) + memcpy(_rules,rules,sizeof(ZT_VirtualNetworkRule) * _ruleCount); + } + + /** + * @return Rules -- see ruleCount() for size of array + */ + inline const ZT_VirtualNetworkRule *rules() const { return _rules; } + + /** + * @return Number of rules in rules() + */ + inline unsigned int ruleCount() const { return _ruleCount; } + + /** + * @return ID and evaluation order of this capability in network + */ + inline uint32_t id() const { return _id; } + + /** + * @return Network ID for which this capability was issued + */ + inline uint64_t networkId() const { return _nwid; } + + /** + * Sign this capability and add signature to its chain of custody + * + * If this returns false, this object should be considered to be + * in an undefined state and should be discarded. False can be returned + * if there is no more room for signatures (max chain length reached) + * or if the 'from' identity does not include a secret key to allow + * it to sign anything. + * + * @param from Signing identity (must have secret) + * @param to Recipient of this signature + * @return True if signature successful and chain of custody appended + */ + inline bool sign(const Identity &from,const Address &to) + { + try { + Buffer<(sizeof(Capability) * 2)> tmp; + for(unsigned int i=0;((i<_maxCustodyChainLength)&&(iserialize(tmp,true); + _custody[i].signature = from.sign(tmp.data(),tmp.size()); + return true; + } + } + } catch ( ... ) {} + return false; + } + + /** + * Verify this capability's chain of custody + * + * This returns a tri-state result. A return value of zero indicates that + * the chain of custody is valid and all signatures are okay. A positive + * return value means at least one WHOIS was issued for a missing signing + * identity and we should retry later. A negative return value means that + * this chain or one of its signature is BAD and this capability should + * be discarded. + * + * Note that the entire chain is checked regardless of verifyInChain. + * + * @param RR Runtime environment to provide for peer lookup, etc. + * @param verifyInChain Also check to ensure that this capability was at some point properly issued to this peer (if non-null) + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain + */ + int verify(const RuntimeEnvironment *RR,const Address &verifyInChain) const; + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_id); + b.append(_nwid); + b.append(_expiration); + + b.append((uint16_t)_ruleCount); + for(unsigned int i=0;i<_ruleCount;++i) { + // Each rule consists of its 8-bit type followed by the size of that type's + // field followed by field data. The inclusion of the size will allow non-supported + // rules to be ignored but still parsed. + b.append((uint8_t)_rules[i].t); + switch((ZT_VirtualNetworkRuleType)(_rules[i].t & 0x7f)) { + //case ZT_NETWORK_RULE_ACTION_DROP: + //case ZT_NETWORK_RULE_ACTION_ACCEPT: + default: + b.append((uint8_t)0); + break; + case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_REDIRECT: + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + b.append((uint8_t)5); + Address(_rules[i].v.zt).appendTo(b); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + b.append((uint8_t)2); + b.append((uint16_t)_rules[i].v.vlanId); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + b.append((uint8_t)1); + b.append((uint8_t)_rules[i].v.vlanPcp); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + b.append((uint8_t)1); + b.append((uint8_t)_rules[i].v.vlanDei); + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + b.append((uint8_t)2); + b.append((uint16_t)_rules[i].v.etherType); + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + b.append((uint8_t)6); + b.append(_rules[i].v.mac,6); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + b.append((uint8_t)5); + b.append(&(_rules[i].v.ipv4.ip),4); + b.append((uint8_t)_rules[i].v.ipv4.mask); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + b.append((uint8_t)17); + b.append(_rules[i].v.ipv6.ip,16); + b.append((uint8_t)_rules[i].v.ipv6.mask); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + b.append((uint8_t)1); + b.append((uint8_t)_rules[i].v.ipTos); + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + b.append((uint8_t)1); + b.append((uint8_t)_rules[i].v.ipProtocol); + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + b.append((uint8_t)4); + b.append((uint16_t)_rules[i].v.port[0]); + b.append((uint16_t)_rules[i].v.port[1]); + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + b.append((uint8_t)16); + b.append((uint64_t)_rules[i].v.characteristics[0]); + b.append((uint64_t)_rules[i].v.characteristics[1]); + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + b.append((uint8_t)4); + b.append((uint16_t)_rules[i].v.frameSize[0]); + b.append((uint16_t)_rules[i].v.frameSize[1]); + break; + } + } + + b.append((uint8_t)_maxCustodyChainLength); + for(unsigned int i=0;;++i) { + if ((i < _maxCustodyChainLength)&&(i < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)&&(_custody[i].to)) { + _custody[i].to.appendTo(b); + _custody[i].from.appendTo(b); + if (!forSign) { + b.append((uint8_t)1); // 1 == Ed25519 signature + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); // length of signature + b.append(_custody[i].signature.data,ZT_C25519_SIGNATURE_LEN); + } + } else { + b.append((unsigned char)0,ZT_ADDRESS_LENGTH); // zero 'to' terminates chain + break; + } + } + + // This is the size of any additional fields. If it is nonzero, + // the last 2 bytes of the next field will be another size field. + b.append((uint16_t)0); + + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + memset(this,0,sizeof(Capability)); + + unsigned int p = startAt; + + _id = b.template at(p); p += 4; + _nwid = b.template at(p); p += 8; + _expiration = b.template at(p); p += 8; + + _ruleCount = b.template at(p); p += 2; + if (_ruleCount > ZT_MAX_CAPABILITY_RULES) + throw std::runtime_error("rule count overflow"); + for(unsigned int i=0;i<_ruleCount;++i) { + _rules[i].t = (uint8_t)b[p++]; + const unsigned int fieldLen = (unsigned int)b[p++]; + switch((ZT_VirtualNetworkRuleType)(_rules[i].t & 0x7f)) { + default: + break; + case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_REDIRECT: + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + _rules[i].v.zt = Address(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + _rules[i].v.vlanId = b.template at(p); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + _rules[i].v.vlanPcp = (uint8_t)b[p]; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + _rules[i].v.vlanDei = (uint8_t)b[p]; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + _rules[i].v.etherType = b.template at(p); + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + memcpy(_rules[i].v.mac,b.field(p,6),6); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + memcpy(&(_rules[i].v.ipv4.ip),b.field(p,4),4); + _rules[i].v.ipv4.mask = (uint8_t)b[p + 4]; + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + memcpy(_rules[i].v.ipv6.ip,b.field(p,16),16); + _rules[i].v.ipv6.mask = (uint8_t)b[p + 16]; + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + _rules[i].v.ipTos = (uint8_t)b[p]; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + _rules[i].v.ipProtocol = (uint8_t)b[p]; + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + _rules[i].v.port[0] = b.template at(p); + _rules[i].v.port[1] = b.template at(p + 2); + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + _rules[i].v.characteristics[0] = b.template at(p); + _rules[i].v.characteristics[1] = b.template at(p + 8); + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + _rules[i].v.frameSize[0] = b.template at(p); + _rules[i].v.frameSize[0] = b.template at(p + 2); + break; + } + p += fieldLen; + } + + _maxCustodyChainLength = (unsigned int)b[p++]; + if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) + throw std::runtime_error("invalid max custody chain length"); + for(unsigned int i;;++i) { + const Address to(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + if (!to) + break; + if ((i >= _maxCustodyChainLength)||(i >= ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) + throw std::runtime_error("unterminated custody chain"); + _custody[i].to = to; + _custody[i].from.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + memcpy(_custody[i].signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + } + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + + // Provides natural sort order by ID + inline bool operator<(const Capability &c) const { return (_id < c._id); } + +private: + uint64_t _nwid; + uint64_t _expiration; + uint32_t _id; + + unsigned int _maxCustodyChainLength; + + unsigned int _ruleCount; + ZT_VirtualNetworkRule _rules[ZT_MAX_CAPABILITY_RULES]; + + struct { + Address to; + Address from; + C25519::Signature signature; + } _custody[ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH]; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 430ed785..8fae8b08 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -316,7 +316,7 @@ public: _signedBy.zero(); if (b[p++] != 1) - throw std::invalid_argument("invalid field type"); + throw std::invalid_argument("invalid object"); unsigned int numq = b.template at(p); p += sizeof(uint16_t); uint64_t lastId = 0; diff --git a/node/Identity.cpp b/node/Identity.cpp index 6f89a1ee..c47805d9 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -133,7 +133,7 @@ std::string Identity::toString(bool includePrivate) const std::string r; r.append(_address.toString()); - r.append(":0:"); // 0 == IDENTITY_TYPE_C25519 + r.append(":0:"); // 0 == ZT_OBJECT_TYPE_IDENTITY r.append(Utils::hex(_publicKey.data,(unsigned int)_publicKey.size())); if ((_privateKey)&&(includePrivate)) { r.push_back(':'); diff --git a/node/Identity.hpp b/node/Identity.hpp index e19c4980..4aa93b87 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -46,14 +46,6 @@ namespace ZeroTier { class Identity { public: - /** - * Identity types - */ - enum Type - { - IDENTITY_TYPE_C25519 = 0 - }; - Identity() : _privateKey((C25519::Private *)0) { @@ -205,11 +197,6 @@ public: return false; } - /** - * @return Identity type - */ - inline Type type() const throw() { return IDENTITY_TYPE_C25519; } - /** * @return This identity's address */ @@ -226,7 +213,7 @@ public: inline void serialize(Buffer &b,bool includePrivate = false) const { _address.appendTo(b); - b.append((unsigned char)IDENTITY_TYPE_C25519); + b.append((uint8_t)0); // C25519/Ed25519 identity type b.append(_publicKey.data,(unsigned int)_publicKey.size()); if ((_privateKey)&&(includePrivate)) { b.append((unsigned char)_privateKey->size()); @@ -257,7 +244,7 @@ public: _address.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - if (b[p++] != IDENTITY_TYPE_C25519) + if (b[p++] != 0) throw std::invalid_argument("unsupported identity type"); memcpy(_publicKey.data,b.field(p,(unsigned int)_publicKey.size()),(unsigned int)_publicKey.size()); diff --git a/node/Packet.hpp b/node/Packet.hpp index 211c3aa5..bd70b6f2 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -690,13 +690,9 @@ public: * controllers and root servers. In the current network, root servers * will provide the service of final multicast cache. * - * It is recommended that NETWORK_MEMBERSHIP_CERTIFICATE pushes be sent - * along with MULTICAST_LIKE when pushing LIKEs to peers that do not - * share a network membership (such as root servers), since this can be - * used to authenticate GATHER requests and limit responses to peers - * authorized to talk on a network. (Should be an optional field here, - * but saving one or two packets every five minutes is not worth an - * ugly hack or protocol rev.) + * If sending LIKEs to root servers for backward compatibility reasons, + * VERB_NETWORK_MEMBERSHIP_CERTIFICATE must be sent as well ahead of + * time so that roots can authenticate GATHER requests. * * OK/ERROR are not generated. */ @@ -720,7 +716,9 @@ public: * /controller/network//member/ * * When received in this manner the response is sent via the old - * OK(NETWORK_CONFIG_REQUEST) instead of OK(REQUEST_OBJECT). + * OK(NETWORK_CONFIG_REQUEST) instead of OK(REQUEST_OBJECT). If the + * response is too large, a dictionary is sent with the single key + * OVF set to 1. In this case REQUEST_OBJECT must be used. * * OK response payload: * <[8] 64-bit network ID> diff --git a/node/Tag.hpp b/node/Tag.hpp new file mode 100644 index 00000000..a4bc4479 --- /dev/null +++ b/node/Tag.hpp @@ -0,0 +1,171 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_TAG_HPP +#define ZT_TAG_HPP + +#include +#include +#include +#include + +#include "Constants.hpp" +#include "C25519.hpp" +#include "Address.hpp" +#include "Identity.hpp" +#include "Buffer.hpp" + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * A tag that can be associated with members and matched in rules + * + * Capabilities group rules, while tags group members subject to those + * rules. Tag values can be matched in rules, and tags relevant to a + * capability are presented along with it. + * + * E.g. a capability might be "can speak Samba/CIFS within your + * department." This cap might have a rule to allow TCP/137 but + * only if a given tag ID's value matches between two peers. The + * capability is what members can do, while the tag is who they are. + * Different departments might have tags with the same ID but different + * values. + * + * Unlike capabilities tags are signed only by the issuer and are never + * transferrable. + */ +class Tag +{ +public: + Tag() + { + memset(this,0,sizeof(Tag)); + } + + /** + * @param nwid Network ID + * @param expiration Tag expiration relative to network config timestamp + * @param issuedTo Address to which this tag was issued + * @param id Tag ID + * @param value Tag value + */ + Tag(const uint64_t nwid,const uint64_t expiration,const Address &issuedTo,const uint32_t id,const uint32_t value) : + _nwid(nwid), + _expiration(expiration), + _id(id), + _value(value), + _issuedTo(issuedTo), + _signedBy() + { + } + + + inline uint64_t networkId() const { return _nwid; } + inline uint64_t expiration() const { return _expiration; } + inline uint32_t id() const { return _id; } + inline uint32_t value() const { return _value; } + inline const Address &issuedTo() const { return _issuedTo; } + inline const Address &signedBy() const { return _signedBy; } + + /** + * Sign this tag + * + * @param signer Signing identity, must have private key + * @return True if signature was successful + */ + inline bool sign(const Identity &signer) + { + try { + Buffer<(sizeof(Tag) * 2)> tmp; + _signedBy = signer.address(); + this->serialize(tmp,true); + _signature = signer.sign(tmp.data(),tmp.size()); + return true; + } catch ( ... ) {} + return false; + } + + /** + * Check this tag's signature + * + * @param RR Runtime environment to allow identity lookup for signedBy + * @return True if signature is present and valid + */ + bool verify(const RuntimeEnvironment *RR); + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_nwid); + b.append(_expiration); + b.append(_id); + b.append(_value); + _issuedTo.appendTo(b); + _signedBy.appendTo(b); + if (!forSign) { + b.append((uint8_t)1); // 1 == Ed25519 + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); // length of signature + b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); + } + b.append((uint16_t)0); // length of additional fields, currently 0 + + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + unsigned int p = startAt; + + _nwid = b.template at(p); p += 8; + _expiration = b.template at(p); p += 8; + _id = b.template at(p); p += 4; + _value = b.template at(p); p += 4; + _issuedTo.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + if (b[p++] != 1) + throw std::runtime_error("unrecognized signature type"); + if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) + throw std::runtime_error("invalid signature length"); + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + +private: + uint64_t _nwid; + uint64_t _expiration; + uint32_t _id; + uint32_t _value; + Address _issuedTo; + Address _signedBy; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/World.hpp b/node/World.hpp index fdada2ad..82ee0d0e 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -164,9 +164,9 @@ public: template inline void serialize(Buffer &b,bool forSign = false) const { - if (forSign) - b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - b.append((uint8_t)0x01); // version -- only one valid value for now + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append((uint8_t)0x01); b.append((uint64_t)_id); b.append((uint64_t)_ts); b.append(_updateSigningKey.data,ZT_C25519_PUBLIC_KEY_LEN); @@ -179,8 +179,8 @@ public: for(std::vector::const_iterator ep(r->stableEndpoints.begin());ep!=r->stableEndpoints.end();++ep) ep->serialize(b); } - if (forSign) - b.append((uint64_t)0xf7f7f7f7f7f7f7f7ULL); + + if (forSign) b.append((uint64_t)0xf7f7f7f7f7f7f7f7ULL); } template @@ -191,7 +191,7 @@ public: _roots.clear(); if (b[p++] != 0x01) - throw std::invalid_argument("invalid World serialized version"); + throw std::invalid_argument("invalid object type"); _id = b.template at(p); p += 8; _ts = b.template at(p); p += 8; -- cgit v1.2.3 From 91940cbcf52c6b09f343e365632b8a1701732099 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 2 Aug 2016 14:40:26 -0700 Subject: Kill network preferred relays -- this feature is gone (and was seldom used anyway) in favor of federation. --- node/NetworkConfig.cpp | 62 ------------------------------ node/NetworkConfig.hpp | 102 ------------------------------------------------- node/Node.cpp | 22 +---------- node/Switch.cpp | 24 +----------- node/Topology.cpp | 10 +---- 5 files changed, 4 insertions(+), 216 deletions(-) (limited to 'node') diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 7a7abdd6..a8ab4dac 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -108,24 +108,6 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (ab.length() > 0) { if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ACTIVE_BRIDGES_OLD,ab.c_str())) return false; } - - std::vector rvec(this->relays()); - std::string rl; - for(std::vector::const_iterator i(rvec.begin());i!=rvec.end();++i) { - if (rl.length() > 0) - rl.push_back(','); - rl.append(i->address.toString()); - if (i->phy4) { - rl.push_back(';'); - rl.append(i->phy4.toString()); - } else if (i->phy6) { - rl.push_back(';'); - rl.append(i->phy6.toString()); - } - } - if (rl.length() > 0) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_RELAYS_OLD,rl.c_str())) return false; - } } #endif // ZT_SUPPORT_OLD_STYLE_NETCONF @@ -164,15 +146,6 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_STATIC_IPS,tmp)) return false; } - tmp.clear(); - for(unsigned int i=0;ipinnedCount;++i) { - this->pinned[i].zt.appendTo(tmp); - this->pinned[i].phy.serialize(tmp); - } - if (tmp.size()) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_PINNED,tmp)) return false; - } - tmp.clear(); for(unsigned int i=0;iruleCount;++i) { tmp.append((uint8_t)rules[i].t); @@ -331,32 +304,6 @@ bool NetworkConfig::fromDictionary(const DictionaryaddSpecialist(Address(f),ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); } } - - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_RELAYS_OLD,tmp2,sizeof(tmp2)) > 0) { - char *saveptr = (char *)0; - for(char *f=Utils::stok(tmp2,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) { - char tmp3[256]; - Utils::scopy(tmp3,sizeof(tmp3),f); - - InetAddress phy; - char *semi = tmp3; - while (*semi) { - if (*semi == ';') { - *semi = (char)0; - ++semi; - phy = InetAddress(semi); - } else ++semi; - } - Address zt(tmp3); - - this->addSpecialist(zt,ZT_NETWORKCONFIG_SPECIALIST_TYPE_NETWORK_PREFERRED_RELAY); - if ((phy)&&(this->pinnedCount < ZT_MAX_NETWORK_PINNED)) { - this->pinned[this->pinnedCount].zt = zt; - this->pinned[this->pinnedCount].phy = phy; - ++this->pinnedCount; - } - } - } #else return false; #endif // ZT_SUPPORT_OLD_STYLE_NETCONF @@ -395,15 +342,6 @@ bool NetworkConfig::fromDictionary(const Dictionarypinned[this->pinnedCount].zt.setTo(tmp.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - p += this->pinned[this->pinnedCount].phy.deserialize(tmp,p); - ++this->pinnedCount; - } - } - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_RULES,tmp)) { unsigned int p = 0; while ((p < tmp.size())&&(ruleCount < ZT_MAX_NETWORK_RULES)) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index f2dab6d3..af7ce93b 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -52,11 +52,6 @@ */ #define ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION 0x0000000000000004ULL -/** - * Device is a network preferred relay - */ -#define ZT_NETWORKCONFIG_SPECIALIST_TYPE_NETWORK_PREFERRED_RELAY 0x0000010000000000ULL - /** * Device is an active bridge */ @@ -110,8 +105,6 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_ROUTES "RT" // static IPs (binary blob) #define ZT_NETWORKCONFIG_DICT_KEY_STATIC_IPS "I" -// pinned address physical route mappings (binary blob) -#define ZT_NETWORKCONFIG_DICT_KEY_PINNED "P" // rules (binary blob) #define ZT_NETWORKCONFIG_DICT_KEY_RULES "R" @@ -147,17 +140,6 @@ namespace ZeroTier { class NetworkConfig { public: - /** - * Network preferred relay with optional physical endpoint addresses - * - * This is used by the convenience relays() method. - */ - struct Relay - { - Address address; - InetAddress phy4,phy6; - }; - /** * Create an instance of a NetworkConfig for the test network ID * @@ -283,43 +265,6 @@ public: return r; } - /** - * Get pinned physical address for a given ZeroTier address, if any - * - * @param zt ZeroTier address - * @param af Address family (e.g. AF_INET) or 0 for the first we find of any type - * @return Physical address, if any - */ - inline InetAddress findPinnedAddress(const Address &zt,unsigned int af) const - { - for(unsigned int i=0;i relays() const - { - std::vector r; - for(unsigned int i=0;i &relays) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), _now(now), - _relays(relays), _world(RR->topology->world()) { } @@ -214,17 +213,6 @@ public: // flapping in Cluster mode. if (RR->topology->amRoot()) return; - - // Check for network preferred relays, also considered 'upstream' and thus always - // pinged to keep links up. If they have stable addresses we will try them there. - for(std::vector::const_iterator r(_relays.begin());r!=_relays.end();++r) { - if (r->address == p->address()) { - stableEndpoint4 = r->phy4; - stableEndpoint6 = r->phy6; - upstream = true; - break; - } - } } if (upstream) { @@ -267,7 +255,6 @@ public: private: const RuntimeEnvironment *RR; uint64_t _now; - const std::vector &_relays; World _world; }; @@ -283,7 +270,6 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB _lastPingCheck = now; // Get relays and networks that need config without leaving the mutex locked - std::vector< NetworkConfig::Relay > networkRelays; std::vector< SharedPtr > needConfig; { Mutex::Lock _l(_networks_m); @@ -291,10 +277,6 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) { needConfig.push_back(n->second); } - if (n->second->hasConfig()) { - std::vector r(n->second->config().relays()); - networkRelays.insert(networkRelays.end(),r.begin(),r.end()); - } } } @@ -303,7 +285,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB (*n)->requestConfiguration(); // Do pings and keepalives - _PingPeersThatNeedPing pfunc(RR,now,networkRelays); + _PingPeersThatNeedPing pfunc(RR,now); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); // Update online status, post status change as event diff --git a/node/Switch.cpp b/node/Switch.cpp index f644774f..41756aa9 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -844,33 +844,11 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid) SharedPtr relay; if (!viaPath) { - if (network) { - unsigned int bestq = ~((unsigned int)0); // max unsigned int since quality is lower==better - unsigned int ptr = 0; - for(;;) { - const Address raddr(network->config().nextRelay(ptr)); - if (raddr) { - SharedPtr rp(RR->topology->getPeer(raddr)); - if (rp) { - const unsigned int q = rp->relayQuality(now); - if (q < bestq) { - bestq = q; - rp.swap(relay); - } - } - } else break; - } - } - - if (!relay) - relay = RR->topology->getBestRoot(); - + relay = RR->topology->getBestRoot(); if ( (!relay) || (!(viaPath = relay->getBestPath(now))) ) return false; } - // viaPath will not be null if we make it here - // Push possible direct paths to us if we are relaying if (relay) { peer->pushDirectPaths(viaPath->localAddress(),viaPath->address(),now,false,( (network)&&(network->isAllowed(peer)) )); viaPath->sent(now); diff --git a/node/Topology.cpp b/node/Topology.cpp index 6e96f2eb..9b434732 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -281,15 +281,7 @@ SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCou bool Topology::isUpstream(const Identity &id) const { - if (isRoot(id)) - return true; - std::vector< SharedPtr > nws(RR->node->allNetworks()); - for(std::vector< SharedPtr >::const_iterator nw(nws.begin());nw!=nws.end();++nw) { - if ((*nw)->config().isRelay(id.address())) { - return true; - } - } - return false; + return isRoot(id); } bool Topology::worldUpdateIfValid(const World &newWorld) -- cgit v1.2.3 From 67cb03742e09f7ad83c2edd80e0a8ffbfcfa6285 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 3 Aug 2016 14:12:38 -0700 Subject: Add tag rules and split out rule serialize/deserialize so the code can be reused. --- include/ZeroTierOne.h | 25 +++++++- node/Capability.hpp | 156 ++++++++++++++++++++++++++++++-------------------- node/Filter.cpp | 5 ++ 3 files changed, 124 insertions(+), 62 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index c4696e7d..db405c08 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -525,7 +525,22 @@ enum ZT_VirtualNetworkRuleType /** * Frame size range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 49 + ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 49, + + /** + * Match a range of tag values (equality match if start==end) + */ + ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE = 50, + + /** + * Match if all bits are set in a tag value + */ + ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL = 51, + + /** + * Match if any bit from a mask is set in a tag value + */ + ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY = 52 }; /** @@ -628,6 +643,14 @@ typedef struct * Ethernet packet size in host byte order (start-end, inclusive) */ uint16_t frameSize[2]; + + /** + * For matching tag values + */ + struct { + uint32_t id; + uint32_t value[2]; // only [0] is used for BITS_ALL or BITS_ANY, [0]-[1] for range + } tag; } v; } ZT_VirtualNetworkRule; diff --git a/node/Capability.hpp b/node/Capability.hpp index 6de4e0a1..82342874 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -158,21 +158,15 @@ public: int verify(const RuntimeEnvironment *RR,const Address &verifyInChain) const; template - inline void serialize(Buffer &b,const bool forSign = false) const + static inline void serializeRules(Buffer &b,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { - if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - - b.append(_id); - b.append(_nwid); - b.append(_expiration); - - b.append((uint16_t)_ruleCount); - for(unsigned int i=0;i<_ruleCount;++i) { + b.append((uint16_t)ruleCount); + for(unsigned int i=0;i + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_id); + b.append(_nwid); + b.append(_expiration); + + serializeRules(b,_rules,_ruleCount); b.append((uint8_t)_maxCustodyChainLength); for(unsigned int i=0;;++i) { @@ -269,79 +287,95 @@ public: } template - inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + static inline void deserializeRules(const Buffer &b,unsigned int &p,ZT_VirtualNetworkRule *rules,unsigned int &ruleCount,const unsigned int maxRuleCount) { - memset(this,0,sizeof(Capability)); - - unsigned int p = startAt; - - _id = b.template at(p); p += 4; - _nwid = b.template at(p); p += 8; - _expiration = b.template at(p); p += 8; - - _ruleCount = b.template at(p); p += 2; - if (_ruleCount > ZT_MAX_CAPABILITY_RULES) + ruleCount = b.template at(p); p += 2; + if (ruleCount > maxRuleCount) throw std::runtime_error("rule count overflow"); - for(unsigned int i=0;i<_ruleCount;++i) { - _rules[i].t = (uint8_t)b[p++]; + for(unsigned int i=0;i(p); + rules[i].v.vlanId = b.template at(p); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - _rules[i].v.vlanPcp = (uint8_t)b[p]; + rules[i].v.vlanPcp = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - _rules[i].v.vlanDei = (uint8_t)b[p]; + rules[i].v.vlanDei = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - _rules[i].v.etherType = b.template at(p); + rules[i].v.etherType = b.template at(p); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: - memcpy(_rules[i].v.mac,b.field(p,6),6); + memcpy(rules[i].v.mac,b.field(p,6),6); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - memcpy(&(_rules[i].v.ipv4.ip),b.field(p,4),4); - _rules[i].v.ipv4.mask = (uint8_t)b[p + 4]; + memcpy(&(rules[i].v.ipv4.ip),b.field(p,4),4); + rules[i].v.ipv4.mask = (uint8_t)b[p + 4]; break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - memcpy(_rules[i].v.ipv6.ip,b.field(p,16),16); - _rules[i].v.ipv6.mask = (uint8_t)b[p + 16]; + memcpy(rules[i].v.ipv6.ip,b.field(p,16),16); + rules[i].v.ipv6.mask = (uint8_t)b[p + 16]; break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - _rules[i].v.ipTos = (uint8_t)b[p]; + rules[i].v.ipTos = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - _rules[i].v.ipProtocol = (uint8_t)b[p]; + rules[i].v.ipProtocol = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - _rules[i].v.port[0] = b.template at(p); - _rules[i].v.port[1] = b.template at(p + 2); + rules[i].v.port[0] = b.template at(p); + rules[i].v.port[1] = b.template at(p + 2); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - _rules[i].v.characteristics[0] = b.template at(p); - _rules[i].v.characteristics[1] = b.template at(p + 8); + rules[i].v.characteristics[0] = b.template at(p); + rules[i].v.characteristics[1] = b.template at(p + 8); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - _rules[i].v.frameSize[0] = b.template at(p); - _rules[i].v.frameSize[0] = b.template at(p + 2); + rules[i].v.frameSize[0] = b.template at(p); + rules[i].v.frameSize[0] = b.template at(p + 2); + break; + case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE: + rules[i].v.tag.id = b.template at(p); + rules[i].v.tag.value[0] = b.template at(p + 4); + rules[i].v.tag.value[1] = b.template at(p + 8); + break; + case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: + case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: + rules[i].v.tag.id = b.template at(p); + rules[i].v.tag.value[0] = b.template at(p + 4); break; } p += fieldLen; } + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + memset(this,0,sizeof(Capability)); + + unsigned int p = startAt; + + _id = b.template at(p); p += 4; + _nwid = b.template at(p); p += 8; + _expiration = b.template at(p); p += 8; + + deserializeRules(b,p,_rules,_ruleCount,ZT_MAX_CAPABILITY_RULES); _maxCustodyChainLength = (unsigned int)b[p++]; if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) diff --git a/node/Filter.cpp b/node/Filter.cpp index a4de7201..d86d1a14 100644 --- a/node/Filter.cpp +++ b/node/Filter.cpp @@ -243,6 +243,11 @@ bool Filter::run( case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; + case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE: + break; + case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: + case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: + break; } // thisSetMatches remains true if the current rule matched... or does NOT match if not bit (0x80) is 1 -- cgit v1.2.3 From 7e6e56e2bce240a8d3a4f2825d3f110109a541b6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 3 Aug 2016 18:04:08 -0700 Subject: Bunch of work on pushing and replication of tags and capabilities, and protocol cleanup. --- include/ZeroTierOne.h | 20 ++--- node/Capability.hpp | 5 ++ node/Filter.cpp | 34 +++++--- node/Filter.hpp | 22 +++-- node/IncomingPacket.cpp | 127 ++++++++++------------------ node/IncomingPacket.hpp | 7 +- node/Membership.hpp | 154 ++++++++++++++++++++++++++++++++++ node/Network.cpp | 2 + node/NetworkConfig.hpp | 78 ++++++++++++------ node/Packet.cpp | 4 +- node/Packet.hpp | 215 +++++++++++++++++++++--------------------------- node/Peer.hpp | 169 +------------------------------------ node/Topology.cpp | 33 +------- service/OneService.cpp | 3 + 14 files changed, 409 insertions(+), 464 deletions(-) create mode 100644 node/Membership.hpp (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index db405c08..9679cf64 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -96,11 +96,6 @@ extern "C" { */ #define ZT_MAX_NETWORK_SPECIALISTS 256 -/** - * Maximum number of static physical to ZeroTier address mappings (typically relays, etc.) - */ -#define ZT_MAX_NETWORK_PINNED 16 - /** * Maximum number of multicast group subscriptions per network */ @@ -111,6 +106,16 @@ extern "C" { */ #define ZT_MAX_NETWORK_RULES 256 +/** + * Maximum number of per-node capabilities per network + */ +#define ZT_MAX_NETWORK_CAPABILITIES 64 + +/** + * Maximum number of per-node tags per network + */ +#define ZT_MAX_NETWORK_TAGS 16 + /** * Maximum number of direct network paths to a given peer */ @@ -126,11 +131,6 @@ extern "C" { */ #define ZT_MAX_CAPABILITY_RULES 64 -/** - * Maximum length of a capbility's short descriptive name - */ -#define ZT_MAX_CAPABILITY_NAME_LENGTH 63 - /** * Global maximum length for capability chain of custody (including initial issue) */ diff --git a/node/Capability.hpp b/node/Capability.hpp index 82342874..d050b2b8 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -109,6 +109,11 @@ public: */ inline uint64_t networkId() const { return _nwid; } + /** + * @return Expiration time relative to network config timestamp + */ + inline uint64_t expiration() const { return _expiration; } + /** * Sign this capability and add signature to its chain of custody * diff --git a/node/Filter.cpp b/node/Filter.cpp index d86d1a14..2980149b 100644 --- a/node/Filter.cpp +++ b/node/Filter.cpp @@ -19,15 +19,8 @@ #include #include "Constants.hpp" -#include "RuntimeEnvironment.hpp" -#include "Address.hpp" -#include "MAC.hpp" -#include "InetAddress.hpp" #include "Filter.hpp" -#include "Packet.hpp" -#include "Switch.hpp" -#include "Topology.hpp" -#include "Node.hpp" +#include "InetAddress.hpp" // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) @@ -61,8 +54,8 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig namespace ZeroTier { bool Filter::run( - const RuntimeEnvironment *RR, const uint64_t nwid, + const bool receiving, const Address &ztSource, const Address &ztDest, const MAC &macSource, @@ -72,8 +65,13 @@ bool Filter::run( const unsigned int etherType, const unsigned int vlanId, const ZT_VirtualNetworkRule *rules, - const unsigned int ruleCount) + const unsigned int ruleCount, + const Tag *tags, + const unsigned int tagCount, + Address &sendCopyOfPacketTo) { + sendCopyOfPacketTo.zero(); + // For each set of rules we start by assuming that they match (since no constraints // yields a 'match all' rule). uint8_t thisSetMatches = 1; @@ -92,6 +90,8 @@ bool Filter::run( // This set did match, so perform action! if (rt != ZT_NETWORK_RULE_ACTION_DROP) { if ((rt == ZT_NETWORK_RULE_ACTION_TEE)||(rt == ZT_NETWORK_RULE_ACTION_REDIRECT)) { + sendCopyOfPacketTo = rules[rn].v.zt; + /* // Tee and redirect both want this frame copied to somewhere else. Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(nwid); @@ -102,6 +102,7 @@ bool Filter::run( outp.append(frameData,frameLen); outp.compress(); RR->sw->send(outp,true,nwid); + */ } // For REDIRECT we will want to DROP at this node. For TEE we ACCEPT at this node but // also forward it along as we just did. @@ -244,9 +245,20 @@ bool Filter::run( thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE: - break; case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: + for(unsigned int i=0;i= rules[rn].v.tag.value[0])&&(tags[i].value() <= rules[rn].v.tag.value[1])); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL) { + thisRuleMatches = (uint8_t)((tags[i].value() & rules[rn].v.tag.value[0]) == rules[rn].v.tag.value[0]); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY) { + thisRuleMatches = (uint8_t)((tags[i].value() & rules[rn].v.tag.value[0]) != 0); + } + break; + } + } break; } diff --git a/node/Filter.hpp b/node/Filter.hpp index f8b66134..06aae55f 100644 --- a/node/Filter.hpp +++ b/node/Filter.hpp @@ -21,15 +21,16 @@ #include +#include + #include "Constants.hpp" #include "../include/ZeroTierOne.h" +#include "Address.hpp" +#include "MAC.hpp" +#include "Tag.hpp" namespace ZeroTier { -class Address; -class RuntimeEnvironment; -class MAC; - /** * Network packet filter for rules engine */ @@ -42,8 +43,8 @@ public: * This returns whether or not the packet should be accepted and may also * take other actions for e.g. the TEE and REDIRECT targets. * - * @param RR ZeroTier runtime environment (context) * @param nwid ZeroTier network ID + * @param receiving True if on receiving side, false on sending side * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -54,10 +55,14 @@ public: * @param vlanId 16-bit VLAN ID * @param rules Pointer to array of rules * @param ruleCount Number of rules + * @param tags Tags associated with this node on this network + * @param tagCount Number of tags + * @param sendCopyOfPacketTo Result parameter: if non-NULL send a copy of this packet to another node + * @return True if packet should be accepted for send or receive */ static bool run( - const RuntimeEnvironment *RR, const uint64_t nwid, + const bool receiving, const Address &ztSource, const Address &ztDest, const MAC &macSource, @@ -67,7 +72,10 @@ public: const unsigned int etherType, const unsigned int vlanId, const ZT_VirtualNetworkRule *rules, - const unsigned int ruleCount); + const unsigned int ruleCount, + const Tag *tags, + const unsigned int tagCount, + Address &sendCopyOfPacketTo); }; } // namespace ZeroTier diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e52b3f91..352e4faa 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -38,6 +38,9 @@ #include "Node.hpp" #include "DeferredPackets.hpp" #include "Filter.hpp" +#include "CertificateOfMembership.hpp" +#include "Capability.hpp" +#include "Tag.hpp" namespace ZeroTier { @@ -106,7 +109,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,bool deferred) case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,peer); case Packet::VERB_ECHO: return _doECHO(RR,peer); case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); - case Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return _doNETWORK_MEMBERSHIP_CERTIFICATE(RR,peer); + case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); @@ -155,22 +158,10 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr break; case Packet::ERROR_IDENTITY_COLLISION: - if (RR->topology->isRoot(peer->identity())) + if (RR->topology->isUpstream(peer->identity())) RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; - case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { - /* Note: certificates are public so it's safe to push them to anyone - * who asks. */ - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->hasConfig())&&(network->config().com)) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE); - network->config().com.serialize(outp); - outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); - } - } break; - case Packet::ERROR_NETWORK_ACCESS_DENIED_: { SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) @@ -218,9 +209,13 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer uint64_t worldTimestamp = 0; { unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); - if (ptr < size()) // ZeroTier One < 1.0.3 did not include physical destination address info + + // Get external surface address if present (was not in old versions) + if (ptr < size()) ptr += externalSurfaceAddress.deserialize(*this,ptr); - if ((ptr + 16) <= size()) { // older versions also did not include World IDs or timestamps + + // Get world ID and world timestamp if present (was not in old versions) + if ((ptr + 16) <= size()) { worldId = at(ptr); ptr += 8; worldTimestamp = at(ptr); } @@ -295,7 +290,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } if (externalSurfaceAddress) - RR->sa->iam(id.address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isRoot(id),RR->node->now()); + RR->sa->iam(id.address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now()); Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_HELLO); @@ -379,13 +374,15 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p return true; } - const bool trusted = RR->topology->isRoot(peer->identity()); - InetAddress externalSurfaceAddress; unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2; - if (ptr < size()) // ZeroTier One < 1.0.3 did not include this field + + // Get reported external surface address if present (was not on old versions) + if (ptr < size()) ptr += externalSurfaceAddress.deserialize(*this,ptr); - if ((trusted)&&((ptr + 2) <= size())) { // older versions also did not include this field, and right now we only use if from a root + + // Handle world updates from root servers if present (was not on old versions) + if (((ptr + 2) <= size())&&(RR->topology->isRoot(peer->identity()))) { World worldUpdate; const unsigned int worldLen = at(ptr); ptr += 2; if (worldLen > 0) { @@ -401,17 +398,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if (externalSurfaceAddress) - RR->sa->iam(peer->address(),_localAddress,_remoteAddress,externalSurfaceAddress,trusted,RR->node->now()); + RR->sa->iam(peer->address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; case Packet::VERB_WHOIS: { - if (RR->topology->isRoot(peer->identity())) { + if (RR->topology->isUpstream(peer->identity())) { const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY); - // Right now we can skip this since OK(WHOIS) is only accepted from - // roots. In the future it should be done if we query less trusted - // sources. - //if (id.locallyValidate()) - RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr(new Peer(RR,RR->identity,id)))); + RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr(new Peer(RR,RR->identity,id)))); } } break; @@ -544,7 +537,6 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { if (!network->isAllowed(peer)) { TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id()); - _sendErrorNeedCertificate(RR,peer,network->id()); return true; } @@ -599,7 +591,6 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

isAllowed(peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id()); - _sendErrorNeedCertificate(RR,peer,network->id()); return true; } @@ -704,20 +695,34 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared return true; } -bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { CertificateOfMembership com; + Capability cap; + Tag tag; - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD; - while (ptr < size()) { - ptr += com.deserialize(*this,ptr); + unsigned int p = ZT_PACKET_IDX_PAYLOAD; + while ((p < size())&&((*this)[p])) { + p += com.deserialize(*this,p); peer->validateAndSetNetworkMembershipCertificate(com.networkId(),com); } + ++p; // skip trailing 0 after COMs if present + + if (p < size()) { // check if new capabilities and tags fields are present + const unsigned int numCapabilities = at(p); p += 2; + for(unsigned int i=0;i(p); p += 2; + for(unsigned int i=0;ireceived(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP); } catch ( ... ) { - TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } return true; } @@ -859,7 +864,6 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // that cert might be what we needed. if (!network->isAllowed(peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id()); - _sendErrorNeedCertificate(RR,peer,network->id()); return true; } @@ -1069,22 +1073,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt // into the one we send along to next hops. const unsigned int lengthOfSignedPortionAndSignature = 29 + vlf; - // Get previous hop's credential, if any - const unsigned int previousHopCredentialLength = at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); - CertificateOfMembership previousHopCom; - if (previousHopCredentialLength >= 1) { - switch((*this)[ZT_PACKET_IDX_PAYLOAD + 31 + vlf]) { - case 0x01: { // network certificate of membership for previous hop - const unsigned int phcl = previousHopCom.deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 32 + vlf); - if (phcl != (previousHopCredentialLength - 1)) { - TRACE("dropped CIRCUIT_TEST from %s(%s): previous hop COM invalid (%u != %u)",source().toString().c_str(),_remoteAddress.toString().c_str(),phcl,(previousHopCredentialLength - 1)); - return true; - } - } break; - default: break; - } - } - vlf += previousHopCredentialLength; + // Add length of second "additional fields" section. + vlf += at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); // Check credentials (signature already verified) NetworkConfig originatorCredentialNetworkConfig; @@ -1166,13 +1156,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt if (breadth > 0) { Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST); outp.append(field(ZT_PACKET_IDX_PAYLOAD,lengthOfSignedPortionAndSignature),lengthOfSignedPortionAndSignature); - const unsigned int previousHopCredentialPos = outp.size(); - outp.append((uint16_t)0); // no previous hop credentials: default - if ((originatorCredentialNetworkConfig)&&(!originatorCredentialNetworkConfig.isPublic())&&(originatorCredentialNetworkConfig.com)) { - outp.append((uint8_t)0x01); // COM - originatorCredentialNetworkConfig.com.serialize(outp); - outp.setAt(previousHopCredentialPos,(uint16_t)(outp.size() - (previousHopCredentialPos + 2))); - } + outp.append((uint16_t)0); // no additional fields if (remainingHopsPtr < size()) outp.append(field(remainingHopsPtr,size() - remainingHopsPtr),size() - remainingHopsPtr); @@ -1241,7 +1225,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const try { // If this were allowed from anyone, it would itself be a DOS vector. Right // now we only allow it from roots and controllers of networks you have joined. - bool allowed = RR->topology->isRoot(peer->identity()); + bool allowed = RR->topology->isUpstream(peer->identity()); if (!allowed) { std::vector< SharedPtr > allNetworks(RR->node->allNetworks()); for(std::vector< SharedPtr >::const_iterator n(allNetworks.begin());n!=allNetworks.end();++n) { @@ -1300,16 +1284,6 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const return true; } -bool IncomingPacket::_doREQUEST_OBJECT(const RuntimeEnvironment *RR,const SharedPtr &peer) -{ - return true; -} - -bool IncomingPacket::_doOBJECT_UPDATED(const RuntimeEnvironment *RR,const SharedPtr &peer) -{ - return true; -} - void IncomingPacket::computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16]) { unsigned char salsabuf[131072]; // 131072 == protocol constant, size of memory buffer for this proof of work function @@ -1388,15 +1362,4 @@ bool IncomingPacket::testSalsa2012Sha512ProofOfWorkResult(unsigned int difficult return true; } -void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr &peer,uint64_t nwid) -{ - Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)verb()); - outp.append(packetId()); - outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); - outp.append(nwid); - outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); -} - } // namespace ZeroTier diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index ab7afd51..bfb30a5e 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -172,7 +172,7 @@ private: bool _doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doECHO(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *RR,const SharedPtr &peer); + bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); @@ -180,11 +180,6 @@ private: bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doREQUEST_OBJECT(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doOBJECT_UPDATED(const RuntimeEnvironment *RR,const SharedPtr &peer); - - // Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to communicate - void _sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr &peer,uint64_t nwid); uint64_t _receiveTime; InetAddress _localAddress; diff --git a/node/Membership.hpp b/node/Membership.hpp new file mode 100644 index 00000000..93d347e7 --- /dev/null +++ b/node/Membership.hpp @@ -0,0 +1,154 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_MEMBERSHIP_HPP +#define ZT_MEMBERSHIP_HPP + +#include + +#include +#include + +#include "Constants.hpp" +#include "../include/ZeroTierOne.h" +#include "CertificateOfMembership.hpp" +#include "Capability.hpp" +#include "Tag.hpp" +#include "Hashtable.hpp" +#include "NetworkConfig.hpp" + +namespace ZeroTier { + +class Peer; + +/** + * Information related to a peer's participation on a network + * + * This structure is not thread-safe and must be locked during use. + */ +class Membership +{ +private: + struct TState + { + TState() : lastPushed(0),lastReceived(0) {} + // Last time we pushed this tag to this peer + uint64_t lastPushed; + // Last time we received this tag from this peer + uint64_t lastReceived; + // Tag from peer + Tag tag; + }; + + struct CState + { + CState() : lastPushed(0),lastReceived(0) {} + // Last time we pushed this capability to this peer + uint64_t lastPushed; + // Last time we received this capability from this peer + uint64_t lastReceived; + // Capability from peer + Capability cap; + }; + +public: + Membership() : + _lastPushedCom(0), + _com(), + _caps(8), + _tags(8) + { + } + + /** + * Send COM and other credentials to this peer if needed + * + * This checks last pushed times for our COM and for other credentials and + * sends VERB_NETWORK_CREDENTIALS if the recipient might need them. + * + * @param peer Peer that "owns" this membership + * @param nconf Network configuration + * @param now Current time + * @param capIds Capability IDs that this peer might need + * @param capCount Number of capability IDs + * @param tagIds Tag IDs that this peer might need + * @param tagCount Number of tag IDs + */ + void sendCredentialsIfNeeded(const Peer &peer,const NetworkConfig &nconf,const uint64_t now,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) const; + + /** + * @param nconf Network configuration + * @param id Tag ID + * @return Pointer to tag or NULL if not found + */ + inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const + { + const TState *t = _tags.get(id); + return ((t) ? (((t->lastReceived != 0)&&(t->tag.expiration() < nconf.timestamp)) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); + } + + /** + * @param nconf Network configuration + * @param id Capablity ID + * @return Pointer to capability or NULL if not found + */ + inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const + { + const CState *c = _caps.get(id); + return ((c) ? (((c->lastReceived != 0)&&(c->cap.expiration() < nconf.timestamp)) ? &(c->cap) : (const Capability *)0) : (const Capability *)0); + } + + /** + * Clean up old or stale entries + */ + inline void clean(const uint64_t now) + { + uint32_t *i = (uint32_t *)0; + CState *cs = (CState *)0; + Hashtable::Iterator csi(_caps); + while (csi.next(i,cs)) { + if ((now - std::max(cs->lastPushed,cs->lastReceived)) > (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 3)) + _caps.erase(*i); + } + + i = (uint32_t *)0; + TState *ts = (TState *)0; + Hashtable::Iterator tsi(_tags); + while (tsi.next(i,ts)) { + if ((now - std::max(ts->lastPushed,ts->lastReceived)) > (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 3)) + _tags.erase(*i); + } + } + +private: + // Last time we pushed our COM to this peer + uint64_t _lastPushedCom; + + // COM from this peer + CertificateOfMembership _com; + + // Capability-related state + Hashtable _caps; + + // Tag-related state + Hashtable _tags; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/Network.cpp b/node/Network.cpp index 25116647..061cca07 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -216,6 +216,8 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MINOR); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,(uint64_t)ZEROTIER_ONE_VERSION_REVISION); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES,(uint64_t)ZT_MAX_NETWORK_RULES); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES,(uint64_t)ZT_MAX_CAPABILITY_RULES); if (controller() == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index af7ce93b..6158c566 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -35,6 +35,8 @@ #include "MulticastGroup.hpp" #include "Address.hpp" #include "CertificateOfMembership.hpp" +#include "Capability.hpp" +#include "Tag.hpp" #include "Dictionary.hpp" /** @@ -76,6 +78,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv" +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES "Mr" +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES "Mcr" // These dictionary keys are short so they don't take up much room. @@ -288,6 +292,32 @@ public: inline bool operator==(const NetworkConfig &nc) const { return (memcmp(this,&nc,sizeof(NetworkConfig)) == 0); } inline bool operator!=(const NetworkConfig &nc) const { return (!(*this == nc)); } + /** + * Add a specialist or mask flags if already present + * + * This masks the existing flags if the specialist is already here or adds + * it otherwise. + * + * @param a Address of specialist + * @param f Flags (OR of specialist role/type flags) + * @return True if successfully masked or added + */ + inline bool addSpecialist(const Address &a,const uint64_t f) + { + const uint64_t aint = a.toInt(); + for(unsigned int i=0;i * <[...] error-dependent payload> */ - VERB_ERROR = 2, + VERB_ERROR = 0x02, /** * Success response: @@ -583,7 +587,7 @@ public: * <[8] in-re packet ID> * <[...] request-specific payload> */ - VERB_OK = 3, + VERB_OK = 0x03, /** * Query an identity by address: @@ -598,7 +602,7 @@ public: * If the address is not found, no response is generated. WHOIS requests * will time out much like ARP requests and similar do in L2. */ - VERB_WHOIS = 4, + VERB_WHOIS = 0x04, /** * Meet another node at a given protocol address: @@ -626,7 +630,7 @@ public: * * No OK or ERROR is generated. */ - VERB_RENDEZVOUS = 5, + VERB_RENDEZVOUS = 0x05, /** * ZT-to-ZT unicast ethernet frame (shortened EXT_FRAME): @@ -642,31 +646,29 @@ public: * ERROR may be generated if a membership certificate is needed for a * closed network. Payload will be network ID. */ - VERB_FRAME = 6, + VERB_FRAME = 0x06, /** * Full Ethernet frame with MAC addressing and optional fields: * <[8] 64-bit network ID> * <[1] flags> - * [<[...] certificate of network membership>] + * [<[...] certificate of network membership (DEPRECATED)>] * <[6] destination MAC or all zero for destination node> * <[6] source MAC or all zero for node of origin> * <[2] 16-bit ethertype> * <[...] ethernet payload> * * Flags: - * 0x01 - Certificate of network membership is attached + * 0x01 - Certificate of network membership attached (DEPRECATED) * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we * want to attach a certificate since FRAME does not support that. * - * Multicast frames may not be sent as EXT_FRAME. - * * ERROR may be generated if a membership certificate is needed for a * closed network. Payload will be network ID. */ - VERB_EXT_FRAME = 7, + VERB_EXT_FRAME = 0x07, /** * ECHO request (a.k.a. ping): @@ -676,7 +678,7 @@ public: * is generated. Response to ECHO requests is optional and ECHO may be * ignored if a node detects a possible flood. */ - VERB_ECHO = 8, + VERB_ECHO = 0x08, /** * Announce interest in multicast group(s): @@ -690,45 +692,76 @@ public: * controllers and root servers. In the current network, root servers * will provide the service of final multicast cache. * - * If sending LIKEs to root servers for backward compatibility reasons, - * VERB_NETWORK_MEMBERSHIP_CERTIFICATE must be sent as well ahead of - * time so that roots can authenticate GATHER requests. + * VERB_NETWORK_CREDENTIALS should be pushed along with this, especially + * if using upstream (e.g. root) nodes as multicast databases. This allows + * GATHERs to be authenticated. * * OK/ERROR are not generated. */ - VERB_MULTICAST_LIKE = 9, + VERB_MULTICAST_LIKE = 0x09, /** - * Network member certificate replication/push: + * Network membership credential push: * <[...] serialized certificate of membership> - * [ ... additional certificates may follow ...] + * [<[...] additional certificates of membership>] + * <[1] null byte for backward compatibility (see below)> + * <[2] 16-bit number of capabilities> + * <[...] one or more serialized Capability> + * <[2] 16-bit number of tags> + * <[...] one or more serialized Tags> * * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may * be pushed at any other time to keep exchanged certificates up to date. * + * Protocol versions prior to 8 do not support capabilities or tags and + * just expect an array of COMs. Adding a single NULL byte after the COM + * array causes these older versions to harmlessly abort parsing and + * ignore the newer fields. The new version checks for this null byte to + * indicate the end of the COM array, since all serialized COMs begin with + * non-zero bytes (see CertificateOfMembership). + * * OK/ERROR are not generated. */ - VERB_NETWORK_MEMBERSHIP_CERTIFICATE = 10, + VERB_NETWORK_CREDENTIALS = 0x0a, /** - * DEPRECATED but still supported, interpreted as an object request: - * - * /controller/network//member/ + * Network configuration request: + * <[8] 64-bit network ID> + * <[2] 16-bit length of request meta-data dictionary> + * <[...] string-serialized request meta-data> + * [<[8] 64-bit timestamp of netconf we currently have>] * - * When received in this manner the response is sent via the old - * OK(NETWORK_CONFIG_REQUEST) instead of OK(REQUEST_OBJECT). If the - * response is too large, a dictionary is sent with the single key - * OVF set to 1. In this case REQUEST_OBJECT must be used. + * This message requests network configuration from a node capable of + * providing it. If the optional revision is included, a response is + * only generated if there is a newer network configuration available. * * OK response payload: * <[8] 64-bit network ID> - * <[2] 16-bit length of network configuration dictionary> - * <[...] network configuration dictionary> + * <[2] 16-bit length of network configuration dictionary field> + * <[...] network configuration dictionary (or fragment)> + * [<[4] 32-bit total length of assembled dictionary>] + * [<[4] 32-bit index of fragment in this reply>] + * + * Fields after the dictionary are extensions to support multipart + * sending of large network configs. If they are not present the + * sent config must be assumed to be whole. * * ERROR response payload: * <[8] 64-bit network ID> */ - VERB_NETWORK_CONFIG_REQUEST = 11, + VERB_NETWORK_CONFIG_REQUEST = 0x0b, + + /** + * Network configuration refresh request: + * <[...] array of 64-bit network IDs> + * + * This can be sent by the network controller to inform a node that it + * should now make a NETWORK_CONFIG_REQUEST. + * + * It does not generate an OK or ERROR message, and is treated only as + * a hint to refresh now. + */ + VERB_NETWORK_CONFIG_REFRESH = 0x0c, /** * Request endpoints for multicast distribution: @@ -737,10 +770,10 @@ public: * <[6] MAC address of multicast group being queried> * <[4] 32-bit ADI for multicast group being queried> * <[4] 32-bit requested max number of multicast peers> - * [<[...] network certificate of membership>] + * [<[...] network certificate of membership (DEPRECATED)>] * * Flags: - * 0x01 - Network certificate of membership is attached + * 0x01 - COM is attached (DEPRECATED) * * This message asks a peer for additional known endpoints that have * LIKEd a given multicast group. It's sent when the sender wishes @@ -750,6 +783,9 @@ public: * More than one OK response can occur if the response is broken up across * multiple packets or if querying a clustered node. * + * Send VERB_NETWORK_CREDENTIALS prior to GATHERing if doing so from + * upstream nodes like root servers that are not involved in our network. + * * OK response payload: * <[8] 64-bit network ID> * <[6] MAC address of multicast group being queried> @@ -761,13 +797,13 @@ public: * * ERROR is not generated; queries that return no response are dropped. */ - VERB_MULTICAST_GATHER = 13, + VERB_MULTICAST_GATHER = 0x0d, /** * Multicast frame: * <[8] 64-bit network ID> * <[1] flags> - * [<[...] network certificate of membership>] + * [<[...] network certificate of membership (DEPRECATED)>] * [<[4] 32-bit implicit gather limit>] * [<[6] source MAC>] * <[6] destination MAC (multicast address)> @@ -776,7 +812,7 @@ public: * <[...] ethernet payload> * * Flags: - * 0x01 - Network certificate of membership is attached + * 0x01 - Network certificate of membership attached (DEPRECATED) * 0x02 - Implicit gather limit field is present * 0x04 - Source MAC is specified -- otherwise it's computed from sender * @@ -791,11 +827,11 @@ public: * <[6] MAC address of multicast group> * <[4] 32-bit ADI for multicast group> * <[1] flags> - * [<[...] network certficate of membership>] + * [<[...] network certficate of membership (DEPRECATED)>] * [<[...] implicit gather results if flag 0x01 is set>] * * OK flags (same bits as request flags): - * 0x01 - OK includes certificate of network membership + * 0x01 - OK includes certificate of network membership (DEPRECATED) * 0x02 - OK includes implicit gather results * * ERROR response payload: @@ -803,7 +839,9 @@ public: * <[6] multicast group MAC> * <[4] 32-bit multicast group ADI> */ - VERB_MULTICAST_FRAME = 14, + VERB_MULTICAST_FRAME = 0x0e, + + // 0x0f is reserved for an old deprecated message /** * Push of potential endpoints for direct communication: @@ -839,7 +877,7 @@ public: * * OK and ERROR are not generated. */ - VERB_PUSH_DIRECT_PATHS = 16, + VERB_PUSH_DIRECT_PATHS = 0x10, /** * Source-routed circuit test message: @@ -855,9 +893,8 @@ public: * [ ... end of signed portion of request ... ] * <[2] 16-bit length of signature of request> * <[...] signature of request by originator> - * <[2] 16-bit previous hop credential length (including type)> - * [[1] previous hop credential type] - * [[...] previous hop credential] + * <[2] 16-bit length of additional fields> + * [[...] additional fields] * <[...] next hop(s) in path> * * Flags: @@ -867,9 +904,6 @@ public: * Originator credential types: * 0x01 - 64-bit network ID for which originator is controller * - * Previous hop credential types: - * 0x01 - Certificate of network membership - * * Path record format: * <[1] 8-bit flags (unused, must be zero)> * <[1] 8-bit breadth (number of next hops)> @@ -918,7 +952,7 @@ public: * <[8] 64-bit timestamp (echoed from original> * <[8] 64-bit test ID (echoed from original)> */ - VERB_CIRCUIT_TEST = 17, + VERB_CIRCUIT_TEST = 0x11, /** * Circuit test hop report: @@ -955,7 +989,7 @@ public: * If a test report is received and no circuit test was sent, it should be * ignored. This message generates no OK or ERROR response. */ - VERB_CIRCUIT_TEST_REPORT = 18, + VERB_CIRCUIT_TEST_REPORT = 0x12, /** * Request proof of work: @@ -998,63 +1032,7 @@ public: * * ERROR has no payload. */ - VERB_REQUEST_PROOF_OF_WORK = 19, - - /** - * Request an object or a chunk of an object with optional meta-data: - * <[8] 64-bit chunk offset> - * <[2] 16-bit chunk length or 0 for any / sender-preferred> - * <[2] 16-bit object path length in bytes> - * <[...] object path> - * <[2] 16-bit length of request meta-data dictionary> - * <[...] request meta-data dictionary> - * - * This is used to request an object. Objects can be things like network - * configs, software updates, etc. This provides an in-band way to - * distribute such things and obsoletes the network config specific - * messages. (They are still supported for backward compatibility.) - * - * The use of path and request/response meta-data makes the semantics of - * this analogous to HTTP POST, and it could therefore be mapped to - * HTTP POST requests to permit plugins that leverage the ZT protocol - * to do out-of-band things like special authentication, etc. - * - * Large objects can be transferred via repeated calls with higher and - * higher chunk offsets and then SHA-512 verified on receipt, but this is - * not efficient. It should not be used heavily as an alternative to - * TCP. It's a bit more like X-Modem and other old-school SEND/ACK - * protocols. It is potentially a good idea for software updates since - * it means that ZT can update itself even on networks with no "vanilla" - * Internet access. - * - * OK and ERROR responses are optional but recommended. ERROR responses - * can include OBJECT_NOT_FOUND. - * - * OK response payload: - * <[16] first 16 bytes of SHA-512 of complete object> - * <[8] 64-bit total object size> - * <[8] 64-bit chunk offset> - * <[2] 16-bit length of chunk payload> - * <[...] chunk payload> - */ - VERB_REQUEST_OBJECT = 20, - - /** - * Notification of a remote object update: - * <[8] 64-bit total object size or 0 if unspecified here> - * <[16] first 16 bytes of SHA-512 of object (if size specified)> - * <[2] 16-bit length of object path> - * <[...] object path> - * <[2] 16-bit length of meta-data dictionary> - * <[...] meta-data dictionary> - * - * This can be sent to notify another peer that an object has updated and - * should be re-requested. The receiving peer is not required to do anything - * or send anything in response to this. If the first size field is zero, the - * SHA-512 hash is also unspecified and should be zero. This means that the - * object was updated but must be re-requested. - */ - VERB_OBJECT_UPDATED = 21 + VERB_REQUEST_PROOF_OF_WORK = 0x13 }; /** @@ -1063,31 +1041,28 @@ public: enum ErrorCode { /* No error, not actually used in transit */ - ERROR_NONE = 0, + ERROR_NONE = 0x00, /* Invalid request */ - ERROR_INVALID_REQUEST = 1, + ERROR_INVALID_REQUEST = 0x01, /* Bad/unsupported protocol version */ - ERROR_BAD_PROTOCOL_VERSION = 2, + ERROR_BAD_PROTOCOL_VERSION = 0x02, /* Unknown object queried */ - ERROR_OBJ_NOT_FOUND = 3, + ERROR_OBJ_NOT_FOUND = 0x03, /* HELLO pushed an identity whose address is already claimed */ - ERROR_IDENTITY_COLLISION = 4, + ERROR_IDENTITY_COLLISION = 0x04, /* Verb or use case not supported/enabled by this node */ - ERROR_UNSUPPORTED_OPERATION = 5, - - /* Message to private network rejected -- no unexpired certificate on file */ - ERROR_NEED_MEMBERSHIP_CERTIFICATE = 6, + ERROR_UNSUPPORTED_OPERATION = 0x05, /* Tried to join network, but you're not a member */ - ERROR_NETWORK_ACCESS_DENIED_ = 7, /* extra _ to avoid Windows name conflict */ + ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */ /* Multicasts to this group are not wanted */ - ERROR_UNWANTED_MULTICAST = 8 + ERROR_UNWANTED_MULTICAST = 0x08 }; //#ifdef ZT_TRACE diff --git a/node/Peer.hpp b/node/Peer.hpp index 445535c8..d8c44ebe 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -31,7 +31,6 @@ #include "../include/ZeroTierOne.h" #include "RuntimeEnvironment.hpp" -#include "CertificateOfMembership.hpp" #include "Path.hpp" #include "Address.hpp" #include "Utils.hpp" @@ -44,10 +43,6 @@ #include "Mutex.hpp" #include "NonCopyable.hpp" -// Very rough computed estimate: (8 + 256 + 80 + (16 * 64) + (128 * 256) + (128 * 16)) -// 1048576 provides tons of headroom -- overflow would just cause peer not to be persisted -#define ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE 1048576 - namespace ZeroTier { /** @@ -362,31 +357,6 @@ public: */ void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; - /** - * Check network COM agreement with this peer - * - * @param nwid Network ID - * @param com Another certificate of membership - * @return True if supplied COM agrees with ours, false if not or if we don't have one - */ - bool networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const; - - /** - * Check the validity of the COM and add/update if valid and new - * - * @param nwid Network ID - * @param com Externally supplied COM - */ - bool validateAndSetNetworkMembershipCertificate(uint64_t nwid,const CertificateOfMembership &com); - - /** - * @param nwid Network ID - * @param now Current time - * @param updateLastPushedTime If true, go ahead and update the last pushed time regardless of return value - * @return Whether or not this peer needs another COM push from us - */ - bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime); - /** * Perform periodic cleaning operations * @@ -434,138 +404,12 @@ public: else return std::pair(); } - template - inline void serialize(Buffer &b) const - { - Mutex::Lock _l(_networkComs_m); - - const unsigned int recSizePos = b.size(); - b.addSize(4); // space for uint32_t field length - - b.append((uint16_t)1); // version of serialized Peer data - - _id.serialize(b,false); - - b.append((uint64_t)_lastUsed); - b.append((uint64_t)_lastReceive); - b.append((uint64_t)_lastUnicastFrame); - b.append((uint64_t)_lastMulticastFrame); - b.append((uint64_t)_lastAnnouncedTo); - b.append((uint64_t)_lastDirectPathPushSent); - b.append((uint64_t)_lastDirectPathPushReceive); - b.append((uint64_t)_lastPathSort); - b.append((uint16_t)_vProto); - b.append((uint16_t)_vMajor); - b.append((uint16_t)_vMinor); - b.append((uint16_t)_vRevision); - b.append((uint32_t)_latency); - b.append((uint16_t)_directPathPushCutoffCount); - - b.append((uint16_t)_numPaths); - for(unsigned int i=0;i<_numPaths;++i) - _paths[i].serialize(b); - - b.append((uint32_t)_networkComs.size()); - { - uint64_t *k = (uint64_t *)0; - _NetworkCom *v = (_NetworkCom *)0; - Hashtable::Iterator i(const_cast(this)->_networkComs); - while (i.next(k,v)) { - b.append((uint64_t)*k); - b.append((uint64_t)v->ts); - v->com.serialize(b); - } - } - - b.append((uint32_t)_lastPushedComs.size()); - { - uint64_t *k = (uint64_t *)0; - uint64_t *v = (uint64_t *)0; - Hashtable::Iterator i(const_cast(this)->_lastPushedComs); - while (i.next(k,v)) { - b.append((uint64_t)*k); - b.append((uint64_t)*v); - } - } - - b.template setAt(recSizePos,(uint32_t)(b.size() - (recSizePos + 4))); // set size - } - - /** - * Create a new Peer from a serialized instance - * - * @param renv Runtime environment - * @param myIdentity This node's identity - * @param b Buffer containing serialized Peer data - * @param p Pointer to current position in buffer, will be updated in place as buffer is read (value/result) - * @return New instance of Peer or NULL if serialized data was corrupt or otherwise invalid (may also throw an exception via Buffer) - */ - template - static inline SharedPtr deserializeNew(const RuntimeEnvironment *renv,const Identity &myIdentity,const Buffer &b,unsigned int &p) - { - const unsigned int recSize = b.template at(p); p += 4; - if ((p + recSize) > b.size()) - return SharedPtr(); // size invalid - if (b.template at(p) != 1) - return SharedPtr(); // version mismatch - p += 2; - - Identity npid; - p += npid.deserialize(b,p); - if (!npid) - return SharedPtr(); - - SharedPtr np(new Peer(renv,myIdentity,npid)); - - np->_lastUsed = b.template at(p); p += 8; - np->_lastReceive = b.template at(p); p += 8; - np->_lastUnicastFrame = b.template at(p); p += 8; - np->_lastMulticastFrame = b.template at(p); p += 8; - np->_lastAnnouncedTo = b.template at(p); p += 8; - np->_lastDirectPathPushSent = b.template at(p); p += 8; - np->_lastDirectPathPushReceive = b.template at(p); p += 8; - np->_lastPathSort = b.template at(p); p += 8; - np->_vProto = b.template at(p); p += 2; - np->_vMajor = b.template at(p); p += 2; - np->_vMinor = b.template at(p); p += 2; - np->_vRevision = b.template at(p); p += 2; - np->_latency = b.template at(p); p += 4; - np->_directPathPushCutoffCount = b.template at(p); p += 2; - - const unsigned int numPaths = b.template at(p); p += 2; - for(unsigned int i=0;i_paths[np->_numPaths++].deserialize(b,p); - } else { - // Skip any paths beyond max, but still read stream - Path foo; - p += foo.deserialize(b,p); - } - } - - const unsigned int numNetworkComs = b.template at(p); p += 4; - for(unsigned int i=0;i_networkComs[b.template at(p)]; p += 8; - c.ts = b.template at(p); p += 8; - p += c.com.deserialize(b,p); - } - - const unsigned int numLastPushed = b.template at(p); p += 4; - for(unsigned int i=0;i(p); p += 8; - const uint64_t ts = b.template at(p); p += 8; - np->_lastPushedComs.set(nwid,ts); - } - - return np; - } - private: void _doDeadPathDetection(Path &p,const uint64_t now); Path *_getBestPath(const uint64_t now); Path *_getBestPath(const uint64_t now,int inetAddressFamily); - unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized + unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; const RuntimeEnvironment *RR; uint64_t _lastUsed; @@ -586,17 +430,6 @@ private: unsigned int _latency; unsigned int _directPathPushCutoffCount; - struct _NetworkCom - { - _NetworkCom() {} - _NetworkCom(uint64_t t,const CertificateOfMembership &c) : ts(t),com(c) {} - uint64_t ts; - CertificateOfMembership com; - }; - Hashtable _networkComs; - Hashtable _lastPushedComs; - Mutex _networkComs_m; - AtomicCounter __refCount; }; diff --git a/node/Topology.cpp b/node/Topology.cpp index 9b434732..725eed31 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -47,36 +47,7 @@ Topology::Topology(const RuntimeEnvironment *renv) : _trustedPathCount(0), _amRoot(false) { - std::string alls(RR->node->dataStoreGet("peers.save")); - const uint8_t *all = reinterpret_cast(alls.data()); - RR->node->dataStoreDelete("peers.save"); - - Buffer *deserializeBuf = new Buffer(); - unsigned int ptr = 0; - while ((ptr + 4) < alls.size()) { - try { - const unsigned int reclen = ( // each Peer serialized record is prefixed by a record length - ((((unsigned int)all[ptr]) & 0xff) << 24) | - ((((unsigned int)all[ptr + 1]) & 0xff) << 16) | - ((((unsigned int)all[ptr + 2]) & 0xff) << 8) | - (((unsigned int)all[ptr + 3]) & 0xff) - ); - unsigned int pos = 0; - deserializeBuf->copyFrom(all + ptr,reclen + 4); - SharedPtr p(Peer::deserializeNew(RR,RR->identity,*deserializeBuf,pos)); - ptr += pos; - if (!p) - break; // stop if invalid records - if (p->address() != RR->identity.address()) - _peers.set(p->address(),p); - } catch ( ... ) { - break; // stop if invalid records - } - } - delete deserializeBuf; - - clean(RR->node->now()); - + // Get cached world if present std::string dsWorld(RR->node->dataStoreGet("world")); World cachedWorld; if (dsWorld.length() > 0) { @@ -87,6 +58,8 @@ Topology::Topology(const RuntimeEnvironment *renv) : cachedWorld = World(); // clear if cached world is invalid } } + + // Use default or cached world depending on which is shinier World defaultWorld; { Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); diff --git a/service/OneService.cpp b/service/OneService.cpp index 13820f5c..460eb1c9 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -705,6 +705,9 @@ public: } authToken = _trimString(authToken); + // Clean up any legacy files if present + OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + "peers.save").c_str()); + _node = new Node( OSUtils::now(), this, -- cgit v1.2.3 From f057bb63cdc4bebc4608f4f2ed6da4656ddbc8a9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 09:02:35 -0700 Subject: More work on tags and capabilities. --- node/Capability.cpp | 52 +++++++++++++++++++++ node/Capability.hpp | 38 +++++++++------ node/CertificateOfMembership.cpp | 33 ++++++-------- node/CertificateOfMembership.hpp | 12 +++-- node/IncomingPacket.cpp | 27 +++++++---- node/LockingPtr.hpp | 99 ++++++++++++++++++++++++++++++++++++++++ node/Membership.hpp | 92 +++++++++++++++++++++++++++++++++++-- node/Peer.hpp | 33 ++++++++++++++ node/Tag.cpp | 45 ++++++++++++++++++ node/Tag.hpp | 11 +++-- node/Topology.cpp | 4 +- objects.mk | 3 ++ 12 files changed, 396 insertions(+), 53 deletions(-) create mode 100644 node/Capability.cpp create mode 100644 node/LockingPtr.hpp create mode 100644 node/Tag.cpp (limited to 'node') diff --git a/node/Capability.cpp b/node/Capability.cpp new file mode 100644 index 00000000..07eb41a9 --- /dev/null +++ b/node/Capability.cpp @@ -0,0 +1,52 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "Capability.hpp" +#include "RuntimeEnvironment.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Switch.hpp" + +namespace ZeroTier { + +int Capability::verify(const RuntimeEnvironment *RR) const +{ + try { + Buffer<(sizeof(Capability) * 2)> tmp; + this->serialize(tmp,true); + for(unsigned int c=0;ctopology->getIdentity(_custody[c].from)); + if (id) { + if (!id.verify(tmp.data(),tmp.size(),_custody[c].signature)) + return -1; + } else { + RR->sw->requestWhois(_custody[c].from); + return 1; + } + } + return 0; + } catch ( ... ) { + return -1; + } +} + +} // namespace ZeroTier diff --git a/node/Capability.hpp b/node/Capability.hpp index d050b2b8..48282708 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -130,11 +130,11 @@ public: inline bool sign(const Identity &from,const Address &to) { try { - Buffer<(sizeof(Capability) * 2)> tmp; for(unsigned int i=0;((i<_maxCustodyChainLength)&&(i tmp; this->serialize(tmp,true); _custody[i].signature = from.sign(tmp.data(),tmp.size()); return true; @@ -145,22 +145,12 @@ public: } /** - * Verify this capability's chain of custody - * - * This returns a tri-state result. A return value of zero indicates that - * the chain of custody is valid and all signatures are okay. A positive - * return value means at least one WHOIS was issued for a missing signing - * identity and we should retry later. A negative return value means that - * this chain or one of its signature is BAD and this capability should - * be discarded. - * - * Note that the entire chain is checked regardless of verifyInChain. + * Verify this capability's chain of custody and signatures * * @param RR Runtime environment to provide for peer lookup, etc. - * @param verifyInChain Also check to ensure that this capability was at some point properly issued to this peer (if non-null) * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain */ - int verify(const RuntimeEnvironment *RR,const Address &verifyInChain) const; + int verify(const RuntimeEnvironment *RR) const; template static inline void serializeRules(Buffer &b,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) @@ -403,9 +393,31 @@ public: return (p - startAt); } + /** + * Check to see if a given address is a 'to' address in the custody chain + * + * This does not actually do certificate checking. That must be done with verify(). + * + * @param a Address to check + * @return True if address is present + */ + inline bool wasIssuedTo(const Address &a) const + { + for(unsigned int i=0;i ZT_NETWORK_COM_MAX_QUALIFIERS)) + return -1; - uint64_t *const buf = new uint64_t[_qualifierCount * 3]; + const Identity id(RR->topology->getIdentity(_signedBy)); + if (!id) { + RR->sw->requestWhois(_signedBy); + return 1; + } + + uint64_t buf[ZT_NETWORK_COM_MAX_QUALIFIERS * 3]; unsigned int ptr = 0; for(unsigned int i=0;i<_qualifierCount;++i) { buf[ptr++] = Utils::hton(_qualifiers[i].id); buf[ptr++] = Utils::hton(_qualifiers[i].value); buf[ptr++] = Utils::hton(_qualifiers[i].maxDelta); } - - bool valid = false; - try { - valid = id.verify(buf,ptr * sizeof(uint64_t),_signature); - delete [] buf; - } catch ( ... ) { - delete [] buf; - } - return valid; + return (id.verify(buf,ptr * sizeof(uint64_t),_signature) ? 0 : -1); } } // namespace ZeroTier diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 8fae8b08..a04f8255 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -46,10 +46,12 @@ /** * Maximum number of qualifiers allowed in a COM (absolute max: 65535) */ -#define ZT_NETWORK_COM_MAX_QUALIFIERS 256 +#define ZT_NETWORK_COM_MAX_QUALIFIERS 8 namespace ZeroTier { +class RuntimeEnvironment; + /** * Certificate of network membership * @@ -275,12 +277,12 @@ public: bool sign(const Identity &with); /** - * Verify certificate against an identity + * Verify this COM and its signature * - * @param id Identity to verify against - * @return True if certificate is signed by this identity and verification was successful + * @param RR Runtime environment for looking up peers + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - bool verify(const Identity &id) const; + int verify(const RuntimeEnvironment *RR) const; /** * @return True if signed diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 352e4faa..6548bda6 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -443,11 +443,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p unsigned int offset = 0; - if ((flags & 0x01) != 0) { - // OK(MULTICAST_FRAME) includes certificate of membership update + if ((flags & 0x01) != 0) { // deprecated but still used by older peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); - peer->validateAndSetNetworkMembershipCertificate(nwid,com); + LockingPtr m = peer->membership(com.networkId(),true); + if (m) m->addCredential(RR,RR->node->now(),com); } if ((flags & 0x02) != 0) { @@ -583,10 +583,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

validateAndSetNetworkMembershipCertificate(network->id(),com); + LockingPtr m = peer->membership(com.networkId(),true); + if (m) m->addCredential(RR,RR->node->now(),com); } if (!network->isAllowed(peer)) { @@ -698,6 +699,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { + const uint64_t now = RR->node->now(); CertificateOfMembership com; Capability cap; Tag tag; @@ -705,7 +707,9 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S unsigned int p = ZT_PACKET_IDX_PAYLOAD; while ((p < size())&&((*this)[p])) { p += com.deserialize(*this,p); - peer->validateAndSetNetworkMembershipCertificate(com.networkId(),com); + LockingPtr m = peer->membership(com.networkId(),true); + if (!m) return true; // sanity check + m->addCredential(RR,now,com); } ++p; // skip trailing 0 after COMs if present @@ -713,10 +717,16 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S const unsigned int numCapabilities = at(p); p += 2; for(unsigned int i=0;i m = peer->membership(cap.networkId(),true); + if (!m) return true; // sanity check + m->addCredential(RR,now,cap); } const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i m = peer->membership(tag.networkId(),true); + if (!m) return true; // sanity check + m->addCredential(RR,now,tag); } } @@ -854,10 +864,11 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // Offset -- size of optional fields added to position of later fields unsigned int offset = 0; - if ((flags & 0x01) != 0) { + if ((flags & 0x01) != 0) { // deprecated but still used by older peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); - peer->validateAndSetNetworkMembershipCertificate(nwid,com); + LockingPtr m = peer->membership(com.networkId(),true); + if (m) m->addCredential(RR,RR->node->now(),com); } // Check membership after we've read any included COM, since diff --git a/node/LockingPtr.hpp b/node/LockingPtr.hpp new file mode 100644 index 00000000..c373129a --- /dev/null +++ b/node/LockingPtr.hpp @@ -0,0 +1,99 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_LOCKINGPTR_HPP +#define ZT_LOCKINGPTR_HPP + +#include "Mutex.hpp" + +namespace ZeroTier { + +/** + * A simple pointer that locks and holds a mutex until destroyed + * + * Care must be taken when using this. It's not very sophisticated and does + * not handle being copied except for the simple return use case. When it is + * copied it hands off the mutex to the copy and clears it in the original, + * meaning that the mutex is unlocked when the last LockingPtr<> in a chain + * of such handoffs is destroyed. If this chain of handoffs "forks" (more than + * one copy is made) then non-determinism may ensue. + * + * This does not delete or do anything else with the pointer. It also does not + * take care of locking the lock. That must be done beforehand. + */ +template +class LockingPtr +{ +public: + LockingPtr() : + _ptr((T *)0), + _lock((Mutex *)0) + { + } + + LockingPtr(T *obj,Mutex *lock) : + _ptr(obj), + _lock(lock) + { + } + + LockingPtr(const LockingPtr &p) : + _ptr(p._ptr), + _lock(p._lock) + { + const_cast(&p)->_lock = (Mutex *)0; + } + + ~LockingPtr() + { + if (_lock) + _lock->unlock(); + } + + inline LockingPtr &operator=(const LockingPtr &p) + { + _ptr = p._ptr; + _lock = p._lock; + const_cast(&p)->_lock = (Mutex *)0; + return *this; + } + + inline operator bool() const throw() { return (_ptr != (T *)0); } + inline T &operator*() const throw() { return *_ptr; } + inline T *operator->() const throw() { return _ptr; } + + /** + * @return Raw pointer to held object + */ + inline T *ptr() const throw() { return _ptr; } + + inline bool operator==(const LockingPtr &sp) const throw() { return (_ptr == sp._ptr); } + inline bool operator!=(const LockingPtr &sp) const throw() { return (_ptr != sp._ptr); } + inline bool operator>(const LockingPtr &sp) const throw() { return (_ptr > sp._ptr); } + inline bool operator<(const LockingPtr &sp) const throw() { return (_ptr < sp._ptr); } + inline bool operator>=(const LockingPtr &sp) const throw() { return (_ptr >= sp._ptr); } + inline bool operator<=(const LockingPtr &sp) const throw() { return (_ptr <= sp._ptr); } + +private: + T *_ptr; + Mutex *_lock; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/Membership.hpp b/node/Membership.hpp index 93d347e7..642d46c6 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -32,9 +32,16 @@ #include "Hashtable.hpp" #include "NetworkConfig.hpp" +// Expiration time for capability and tag cache +#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 4) + +// Expiration time for Memberships (used in Peer::clean()) +#define ZT_MEMBERSHIP_EXPIRATION_TIME (ZT_MEMBERSHIP_STATE_EXPIRATION_TIME * 4) + namespace ZeroTier { class Peer; +class RuntimeEnvironment; /** * Information related to a peer's participation on a network @@ -81,15 +88,17 @@ public: * This checks last pushed times for our COM and for other credentials and * sends VERB_NETWORK_CREDENTIALS if the recipient might need them. * + * @param RR Runtime environment + * @param now Current time * @param peer Peer that "owns" this membership * @param nconf Network configuration - * @param now Current time * @param capIds Capability IDs that this peer might need * @param capCount Number of capability IDs * @param tagIds Tag IDs that this peer might need * @param tagCount Number of tag IDs + * @return True if we pushed something */ - void sendCredentialsIfNeeded(const Peer &peer,const NetworkConfig &nconf,const uint64_t now,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) const; + bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) const; /** * @param nconf Network configuration @@ -113,26 +122,99 @@ public: return ((c) ? (((c->lastReceived != 0)&&(c->cap.expiration() < nconf.timestamp)) ? &(c->cap) : (const Capability *)0) : (const Capability *)0); } + /** + * Validate and add a credential if signature is okay and it's otherwise good + * + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + inline int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com) + { + if (com.issuedTo() != RR->identity.address()) + return -1; + if (_com == com) + return 0; + const int vr = com.verify(RR); + if (vr == 0) + _com = com; + return vr; + } + + /** + * Validate and add a credential if signature is okay and it's otherwise good + * + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + inline int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Tag &tag) + { + if (tag.issuedTo() != RR->identity.address()) + return -1; + TState *t = _tags.get(tag.networkId()); + if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) + return 0; + const int vr = tag.verify(RR); + if (vr == 0) { + if (!t) + t = &(_tags[tag.networkId()]); + t->lastReceived = now; + t->tag = tag; + } + return vr; + } + + /** + * Validate and add a credential if signature is okay and it's otherwise good + * + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + inline int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Capability &cap) + { + if (!cap.wasIssuedTo(RR->identity.address())) + return -1; + CState *c = _caps.get(cap.networkId()); + if ((c)&&(c->lastReceived != 0)&&(c->cap == cap)) + return 0; + const int vr = cap.verify(RR); + if (vr == 0) { + if (!c) + c = &(_caps[cap.networkId()]); + c->lastReceived = now; + c->cap = cap; + } + return vr; + } + /** * Clean up old or stale entries + * + * @return Time of most recent activity in this Membership */ - inline void clean(const uint64_t now) + inline uint64_t clean(const uint64_t now) { + uint64_t lastAct = _lastPushedCom; + uint32_t *i = (uint32_t *)0; CState *cs = (CState *)0; Hashtable::Iterator csi(_caps); while (csi.next(i,cs)) { - if ((now - std::max(cs->lastPushed,cs->lastReceived)) > (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 3)) + const uint64_t la = std::max(cs->lastPushed,cs->lastReceived); + if ((now - la) > ZT_MEMBERSHIP_STATE_EXPIRATION_TIME) _caps.erase(*i); + else if (la > lastAct) + lastAct = la; } i = (uint32_t *)0; TState *ts = (TState *)0; Hashtable::Iterator tsi(_tags); while (tsi.next(i,ts)) { - if ((now - std::max(ts->lastPushed,ts->lastReceived)) > (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 3)) + const uint64_t la = std::max(ts->lastPushed,ts->lastReceived); + if ((now - la) > ZT_MEMBERSHIP_STATE_EXPIRATION_TIME) _tags.erase(*i); + else if (la > lastAct) + lastAct = la; } + + return lastAct; } private: diff --git a/node/Peer.hpp b/node/Peer.hpp index d8c44ebe..8b50f429 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -40,8 +40,10 @@ #include "SharedPtr.hpp" #include "AtomicCounter.hpp" #include "Hashtable.hpp" +#include "Membership.hpp" #include "Mutex.hpp" #include "NonCopyable.hpp" +#include "LockingPtr.hpp" namespace ZeroTier { @@ -384,6 +386,34 @@ public: return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT); } + /** + * Get the membership record for this network, possibly creating if missing + * + * @param networkId Network ID + * @param createIfMissing If true, create a Membership record if there isn't one + * @return Single-scope locking pointer (see LockingPtr.hpp) to Membership or NULL if not found and createIfMissing is false + */ + inline LockingPtr membership(const uint64_t networkId,bool createIfMissing) + { + _memberships_m.lock(); + try { + if (createIfMissing) { + return LockingPtr(&(_memberships[networkId]),&_memberships_m); + } else { + Membership *m = _memberships.get(networkId); + if (m) { + return LockingPtr(m,&_memberships_m); + } else { + _memberships_m.unlock(); + return LockingPtr(); + } + } + } catch ( ... ) { + _memberships_m.unlock(); + throw; + } + } + /** * Find a common set of addresses by which two peers can link, if any * @@ -430,6 +460,9 @@ private: unsigned int _latency; unsigned int _directPathPushCutoffCount; + Hashtable _memberships; + Mutex _memberships_m; + AtomicCounter __refCount; }; diff --git a/node/Tag.cpp b/node/Tag.cpp new file mode 100644 index 00000000..1ad17251 --- /dev/null +++ b/node/Tag.cpp @@ -0,0 +1,45 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "Tag.hpp" +#include "RuntimeEnvironment.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Switch.hpp" + +namespace ZeroTier { + +int Tag::verify(const RuntimeEnvironment *RR) const +{ + if (!_signedBy) + return -1; + const Identity id(RR->topology->getIdentity(_signedBy)); + if (!id) { + RR->sw->requestWhois(_signedBy); + return 1; + } + try { + Buffer<(sizeof(Tag) * 2)> tmp; + this->serialize(tmp,true); + return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1); + } catch ( ... ) { + return -1; + } +} + +} // namespace ZeroTier diff --git a/node/Tag.hpp b/node/Tag.hpp index a4bc4479..dcf2eb20 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -76,7 +76,6 @@ public: { } - inline uint64_t networkId() const { return _nwid; } inline uint64_t expiration() const { return _expiration; } inline uint32_t id() const { return _id; } @@ -106,9 +105,9 @@ public: * Check this tag's signature * * @param RR Runtime environment to allow identity lookup for signedBy - * @return True if signature is present and valid + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or tag */ - bool verify(const RuntimeEnvironment *RR); + int verify(const RuntimeEnvironment *RR) const; template inline void serialize(Buffer &b,const bool forSign = false) const @@ -156,6 +155,12 @@ public: return (p - startAt); } + // Provides natural sort order by ID + inline bool operator<(const Tag &t) const { return (_id < t._id); } + + inline bool operator==(const Tag &t) const { return (memcmp(this,&t,sizeof(Tag)) == 0); } + inline bool operator!=(const Tag &t) const { return (memcmp(this,&t,sizeof(Tag)) != 0); } + private: uint64_t _nwid; uint64_t _expiration; diff --git a/node/Topology.cpp b/node/Topology.cpp index 725eed31..ef1c1698 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -169,7 +169,9 @@ SharedPtr Topology::getPeer(const Address &zta) Identity Topology::getIdentity(const Address &zta) { - { + if (zta == RR->identity.address()) { + return RR->identity; + } else { Mutex::Lock _l(_lock); const SharedPtr *const ap = _peers.get(zta); if (ap) diff --git a/objects.mk b/objects.mk index da4fda1c..18e330b3 100644 --- a/objects.mk +++ b/objects.mk @@ -1,5 +1,6 @@ OBJS=\ node/C25519.o \ + node/Capability.o \ node/CertificateOfMembership.o \ node/Cluster.o \ node/DeferredPackets.o \ @@ -7,6 +8,7 @@ OBJS=\ node/Identity.o \ node/IncomingPacket.o \ node/InetAddress.o \ + node/Membership.o \ node/Multicaster.o \ node/Network.o \ node/NetworkConfig.o \ @@ -20,6 +22,7 @@ OBJS=\ node/SelfAwareness.o \ node/SHA512.o \ node/Switch.o \ + node/Tag.o \ node/Topology.o \ node/Utils.o \ osdep/BackgroundResolver.o \ -- cgit v1.2.3 From 404a0bbddd90ef2fbd624a0089076aac7e9184b4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 09:51:15 -0700 Subject: ... --- node/Constants.hpp | 11 +--- node/IncomingPacket.cpp | 7 +-- node/Membership.cpp | 133 ++++++++++++++++++++++++++++++++++++++++++++++++ node/Membership.hpp | 62 +++++++--------------- node/NetworkConfig.hpp | 18 +++++++ node/Packet.hpp | 9 +--- 6 files changed, 176 insertions(+), 64 deletions(-) create mode 100644 node/Membership.cpp (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index dc36b3a1..489203fe 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -309,13 +309,6 @@ */ #define ZT_NAT_T_TACTICAL_ESCALATION_DELAY 1000 -/** - * How long (max) to remember network certificates of membership? - * - * This only applies to networks we don't belong to. - */ -#define ZT_PEER_NETWORK_COM_EXPIRATION 3600000 - /** * Sanity limit on maximum bridge routes * @@ -330,7 +323,7 @@ /** * If there is no known route, spam to up to this many active bridges */ -#define ZT_MAX_BRIDGE_SPAM 16 +#define ZT_MAX_BRIDGE_SPAM 32 /** * Interval between direct path pushes in milliseconds @@ -357,7 +350,7 @@ #define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 4 /** - * Enable support for old Dictionary based network configs + * Enable support for older network configurations from older (pre-1.1.6) controllers */ #define ZT_SUPPORT_OLD_STYLE_NETCONF 1 diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 6548bda6..c7e6e439 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -709,7 +709,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += com.deserialize(*this,p); LockingPtr m = peer->membership(com.networkId(),true); if (!m) return true; // sanity check - m->addCredential(RR,now,com); + if (m->addCredential(RR,now,com) == 1) return false; // wait for WHOIS } ++p; // skip trailing 0 after COMs if present @@ -719,14 +719,15 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += cap.deserialize(*this,p); LockingPtr m = peer->membership(cap.networkId(),true); if (!m) return true; // sanity check - m->addCredential(RR,now,cap); + if (m->addCredential(RR,now,cap) == 1) return false; // wait for WHOIS } + const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i m = peer->membership(tag.networkId(),true); if (!m) return true; // sanity check - m->addCredential(RR,now,tag); + if (m->addCredential(RR,now,tag) == 1) return false; // wait for WHOIS } } diff --git a/node/Membership.cpp b/node/Membership.cpp new file mode 100644 index 00000000..91cf693a --- /dev/null +++ b/node/Membership.cpp @@ -0,0 +1,133 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "Membership.hpp" +#include "RuntimeEnvironment.hpp" +#include "Peer.hpp" +#include "Topology.hpp" +#include "Switch.hpp" +#include "Packet.hpp" +#include "Node.hpp" + +#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 2) + +namespace ZeroTier { + +bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) +{ + try { + Buffer capsAndTags; + + capsAndTags.addSize(2); + unsigned int appendedCaps = 0; + for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { + if ((capsAndTags.size() + sizeof(Capability)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) + break; + const Capability *c = nconf.capability(capIds[i]); + if (c) { + c->serialize(capsAndTags); + ++appendedCaps; + cs->lastPushed = now; + } + } + } + capsAndTags.setAt(0,(uint16_t)appendedCaps); + + const unsigned int tagCountPos = capsAndTags.size(); + capsAndTags.addSize(2); + unsigned int appendedTags = 0; + for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { + if ((capsAndTags.size() + sizeof(Tag)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) + break; + const Tag *t = nconf.tag(tagIds[i]); + if (t) { + t->serialize(capsAndTags); + ++appendedTags; + ts->lastPushed = now; + } + } + } + capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); + + if (((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)||(appendedCaps)||(appendedTags)) { + Packet outp(peer.address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + nconf.com.serialize(outp); + outp.append((uint8_t)0x00); + outp.append(capsAndTags.data(),capsAndTags.size()); + outp.compress(); + RR->sw->send(outp,true,0); + _lastPushedCom = now; + return true; + } + } catch ( ... ) { + TRACE("unable to send credentials due to unexpected exception"); + return false; + } +} + +int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com) +{ + if (com.issuedTo() != RR->identity.address()) + return -1; + if (_com == com) + return 0; + const int vr = com.verify(RR); + if (vr == 0) + _com = com; + return vr; +} + +int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Tag &tag) +{ + if (tag.issuedTo() != RR->identity.address()) + return -1; + TState *t = _tags.get(tag.networkId()); + if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) + return 0; + const int vr = tag.verify(RR); + if (vr == 0) { + if (!t) + t = &(_tags[tag.networkId()]); + t->lastReceived = now; + t->tag = tag; + } + return vr; +} + +int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Capability &cap) +{ + if (!cap.wasIssuedTo(RR->identity.address())) + return -1; + CState *c = _caps.get(cap.networkId()); + if ((c)&&(c->lastReceived != 0)&&(c->cap == cap)) + return 0; + const int vr = cap.verify(RR); + if (vr == 0) { + if (!c) + c = &(_caps[cap.networkId()]); + c->lastReceived = now; + c->cap = cap; + } + return vr; +} + +} // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 642d46c6..abfff9e3 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -98,7 +98,21 @@ public: * @param tagCount Number of tag IDs * @return True if we pushed something */ - bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) const; + bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount); + + /** + * Send COM if needed + * + * @param RR Runtime environment + * @param now Current time + * @param peer Peer that "owns" this membership + * @param nconf Network configuration + * @return True if we pushed something + */ + inline bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf) + { + return sendCredentialsIfNeeded(RR,now,peer,nconf,(const uint32_t *)0,0,(const uint32_t *)0,0); + } /** * @param nconf Network configuration @@ -127,61 +141,21 @@ public: * * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - inline int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com) - { - if (com.issuedTo() != RR->identity.address()) - return -1; - if (_com == com) - return 0; - const int vr = com.verify(RR); - if (vr == 0) - _com = com; - return vr; - } + int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good * * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - inline int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Tag &tag) - { - if (tag.issuedTo() != RR->identity.address()) - return -1; - TState *t = _tags.get(tag.networkId()); - if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) - return 0; - const int vr = tag.verify(RR); - if (vr == 0) { - if (!t) - t = &(_tags[tag.networkId()]); - t->lastReceived = now; - t->tag = tag; - } - return vr; - } + int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Tag &tag); /** * Validate and add a credential if signature is okay and it's otherwise good * * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - inline int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Capability &cap) - { - if (!cap.wasIssuedTo(RR->identity.address())) - return -1; - CState *c = _caps.get(cap.networkId()); - if ((c)&&(c->lastReceived != 0)&&(c->cap == cap)) - return 0; - const int vr = cap.verify(RR); - if (vr == 0) { - if (!c) - c = &(_caps[cap.networkId()]); - c->lastReceived = now; - c->cap = cap; - } - return vr; - } + int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Capability &cap); /** * Clean up old or stale entries diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 6158c566..7bbedf20 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -318,6 +318,24 @@ public: return false; } + const Capability *capability(const uint32_t id) const + { + for(unsigned int i=0;i * [<[...] additional certificates of membership>] - * <[1] null byte for backward compatibility (see below)> + * <[1] 0x00, null byte marking end of COM array> * <[2] 16-bit number of capabilities> * <[...] one or more serialized Capability> * <[2] 16-bit number of tags> @@ -713,13 +713,6 @@ public: * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may * be pushed at any other time to keep exchanged certificates up to date. * - * Protocol versions prior to 8 do not support capabilities or tags and - * just expect an array of COMs. Adding a single NULL byte after the COM - * array causes these older versions to harmlessly abort parsing and - * ignore the newer fields. The new version checks for this null byte to - * indicate the end of the COM array, since all serialized COMs begin with - * non-zero bytes (see CertificateOfMembership). - * * OK/ERROR are not generated. */ VERB_NETWORK_CREDENTIALS = 0x0a, -- cgit v1.2.3 From 5cf410490e677f524eda5fd5c790e37f81ba7753 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 10:18:33 -0700 Subject: . --- node/CertificateOfMembership.cpp | 3 ++ node/IncomingPacket.cpp | 12 +++---- node/Membership.hpp | 5 +++ node/Network.cpp | 69 +++++++++++++++++++++------------------- node/Topology.hpp | 8 +++++ 5 files changed, 58 insertions(+), 39 deletions(-) (limited to 'node') diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 7b99f2c7..0c36aa45 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -155,6 +155,9 @@ bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other) c unsigned int myidx = 0; unsigned int otheridx = 0; + if ((_qualifierCount == 0)||(other._qualifierCount == 0)) + return false; + while (myidx < _qualifierCount) { // Fail if we're at the end of other, since this means the field is // missing. diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c7e6e439..029570f1 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -446,7 +446,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p if ((flags & 0x01) != 0) { // deprecated but still used by older peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); - LockingPtr m = peer->membership(com.networkId(),true); + LockingPtr m(peer->membership(com.networkId(),true)); if (m) m->addCredential(RR,RR->node->now(),com); } @@ -586,7 +586,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

m = peer->membership(com.networkId(),true); + LockingPtr m(peer->membership(com.networkId(),true)); if (m) m->addCredential(RR,RR->node->now(),com); } @@ -707,7 +707,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S unsigned int p = ZT_PACKET_IDX_PAYLOAD; while ((p < size())&&((*this)[p])) { p += com.deserialize(*this,p); - LockingPtr m = peer->membership(com.networkId(),true); + LockingPtr m(peer->membership(com.networkId(),true)); if (!m) return true; // sanity check if (m->addCredential(RR,now,com) == 1) return false; // wait for WHOIS } @@ -717,7 +717,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S const unsigned int numCapabilities = at(p); p += 2; for(unsigned int i=0;i m = peer->membership(cap.networkId(),true); + LockingPtr m(peer->membership(cap.networkId(),true)); if (!m) return true; // sanity check if (m->addCredential(RR,now,cap) == 1) return false; // wait for WHOIS } @@ -725,7 +725,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i m = peer->membership(tag.networkId(),true); + LockingPtr m(peer->membership(tag.networkId(),true)); if (!m) return true; // sanity check if (m->addCredential(RR,now,tag) == 1) return false; // wait for WHOIS } @@ -868,7 +868,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((flags & 0x01) != 0) { // deprecated but still used by older peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); - LockingPtr m = peer->membership(com.networkId(),true); + LockingPtr m(peer->membership(com.networkId(),true)); if (m) m->addCredential(RR,RR->node->now(),com); } diff --git a/node/Membership.hpp b/node/Membership.hpp index abfff9e3..0e72b7b1 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -114,6 +114,11 @@ public: return sendCredentialsIfNeeded(RR,now,peer,nconf,(const uint32_t *)0,0,(const uint32_t *)0,0); } + /** + * @return This peer's COM if they have sent one + */ + inline const CertificateOfMembership &com() const { return _com; } + /** * @param nconf Network configuration * @param id Tag ID diff --git a/node/Network.cpp b/node/Network.cpp index 061cca07..d9ad7838 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -29,6 +29,7 @@ #include "Buffer.hpp" #include "NetworkController.hpp" #include "Node.hpp" +#include "Peer.hpp" #include "../version.h" @@ -384,17 +385,20 @@ bool Network::_isAllowed(const SharedPtr &peer) const { // Assumes _lock is locked try { - if (!_config) - return false; - if (_config.isPublic()) - return true; - return ((_config.com)&&(peer->networkMembershipCertificatesAgree(_id,_config.com))); - } catch (std::exception &exc) { - TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer->address().toString().c_str(),exc.what()); + if (_config) { + if (_config.isPublic()) { + return true; + } else { + LockingPtr m(peer->membership(_id,false)); + if (m) { + return _config.com.agreesWith(m->com()); + } + } + } } catch ( ... ) { - TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer->address().toString().c_str()); + TRACE("isAllowed() check failed for peer %s: unexpected exception: unexpected exception",peer->address().toString().c_str()); } - return false; // default position on any failure + return false; } class _MulticastAnnounceAll @@ -405,13 +409,13 @@ public: _controller(nw->controller()), _network(nw), _anchors(nw->config().anchors()), - _rootAddresses(renv->topology->rootAddresses()) + _upstreamAddresses(renv->topology->upstreamAddresses()) {} inline void operator()(Topology &t,const SharedPtr &p) { - if ( (_network->_isAllowed(p)) || // FIXME: this causes multicast LIKEs for public networks to get spammed + if ( (_network->_isAllowed(p)) || // FIXME: this causes multicast LIKEs for public networks to get spammed, which isn't terrible but is a bit stupid (p->address() == _controller) || - (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) || + (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),p->address()) != _upstreamAddresses.end()) || (std::find(_anchors.begin(),_anchors.end(),p->address()) != _anchors.end()) ) { peers.push_back(p); } @@ -422,7 +426,7 @@ private: const Address _controller; Network *const _network; const std::vector

_anchors; - const std::vector
_rootAddresses; + const std::vector
_upstreamAddresses; }; void Network::_announceMulticastGroups() { @@ -438,31 +442,30 @@ void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std:: { // Assumes _lock is locked - // We push COMs ahead of MULTICAST_LIKE since they're used for access control -- a COM is a public - // credential so "over-sharing" isn't really an issue (and we only do so with roots). - if ((_config)&&(_config.com)&&(!_config.isPublic())&&(peer->needsOurNetworkMembershipCertificate(_id,RR->node->now(),true))) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE); - _config.com.serialize(outp); - RR->sw->send(outp,true,0); - } - + // Anyone we announce multicast groups to will need our COM to authenticate GATHER requests. { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); + LockingPtr m(peer->membership(_id,false)); + if (m) m->sendCredentialsIfNeeded(RR,RR->node->now(),*peer,_config); + } - for(std::vector::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { - if ((outp.size() + 18) >= ZT_UDP_DEFAULT_PAYLOAD_MTU) { - RR->sw->send(outp,true,0); - outp.reset(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); - } + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); - // network ID, MAC, ADI - outp.append((uint64_t)_id); - mg->mac().appendTo(outp); - outp.append((uint32_t)mg->adi()); + for(std::vector::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { + if ((outp.size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { + outp.compress(); + RR->sw->send(outp,true,0); + outp.reset(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); } - if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) - RR->sw->send(outp,true,0); + // network ID, MAC, ADI + outp.append((uint64_t)_id); + mg->mac().appendTo(outp); + outp.append((uint32_t)mg->adi()); + } + + if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) { + outp.compress(); + RR->sw->send(outp,true,0); } } diff --git a/node/Topology.hpp b/node/Topology.hpp index 03c491e5..b8213cf8 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -153,6 +153,14 @@ public: return _rootAddresses; } + /** + * @return Vector of active upstream addresses (including roots) + */ + inline std::vector
upstreamAddresses() const + { + return rootAddresses(); + } + /** * @return Current World (copy) */ -- cgit v1.2.3 From 56febbf2bac2c51d9478616a1dd28243ef03f406 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 10:39:28 -0700 Subject: . --- node/Capability.cpp | 25 ++++++---- node/CertificateOfMembership.cpp | 3 +- node/Membership.cpp | 2 +- node/Peer.cpp | 99 +++------------------------------------- node/Tag.cpp | 3 +- 5 files changed, 29 insertions(+), 103 deletions(-) (limited to 'node') diff --git a/node/Capability.cpp b/node/Capability.cpp index 07eb41a9..ee798a6c 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -21,19 +21,29 @@ #include "Identity.hpp" #include "Topology.hpp" #include "Switch.hpp" +#include "Network.hpp" namespace ZeroTier { int Capability::verify(const RuntimeEnvironment *RR) const { try { + if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) + return -1; + Buffer<(sizeof(Capability) * 2)> tmp; this->serialize(tmp,true); - for(unsigned int c=0;ctopology->getIdentity(_custody[c].from)); if (id) { if (!id.verify(tmp.data(),tmp.size(),_custody[c].signature)) @@ -44,9 +54,8 @@ int Capability::verify(const RuntimeEnvironment *RR) const } } return 0; - } catch ( ... ) { - return -1; - } + } catch ( ... ) {} + return -1; } } // namespace ZeroTier diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 0c36aa45..43efcd20 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -20,6 +20,7 @@ #include "RuntimeEnvironment.hpp" #include "Topology.hpp" #include "Switch.hpp" +#include "Network.hpp" namespace ZeroTier { @@ -208,7 +209,7 @@ bool CertificateOfMembership::sign(const Identity &with) int CertificateOfMembership::verify(const RuntimeEnvironment *RR) const { - if ((!_signedBy)||(_qualifierCount > ZT_NETWORK_COM_MAX_QUALIFIERS)) + if ((!_signedBy)||(_signedBy != Network::controllerFor(networkId()))||(_qualifierCount > ZT_NETWORK_COM_MAX_QUALIFIERS)) return -1; const Identity id(RR->topology->getIdentity(_signedBy)); diff --git a/node/Membership.cpp b/node/Membership.cpp index 91cf693a..a1a8eb8a 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -80,8 +80,8 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } } catch ( ... ) { TRACE("unable to send credentials due to unexpected exception"); - return false; } + return false; } int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com) diff --git a/node/Peer.cpp b/node/Peer.cpp index cc581004..a994c4b2 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -371,80 +371,6 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) } } -bool Peer::networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const -{ - Mutex::Lock _l(_networkComs_m); - const _NetworkCom *ourCom = _networkComs.get(nwid); - if (ourCom) - return ourCom->com.agreesWith(com); - return false; -} - -bool Peer::validateAndSetNetworkMembershipCertificate(uint64_t nwid,const CertificateOfMembership &com) -{ - // Sanity checks - if ((!com)||(com.issuedTo() != _id.address())) - return false; - - // Return true if we already have this *exact* COM - { - Mutex::Lock _l(_networkComs_m); - _NetworkCom *ourCom = _networkComs.get(nwid); - if ((ourCom)&&(ourCom->com == com)) - return true; - } - - // Check signature, log and return if cert is invalid - if (com.signedBy() != Network::controllerFor(nwid)) { - TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)nwid,com.signedBy().toString().c_str()); - return false; // invalid signer - } - - if (com.signedBy() == RR->identity.address()) { - - // We are the controller: RR->identity.address() == controller() == cert.signedBy() - // So, verify that we signed th cert ourself - if (!com.verify(RR->identity)) { - TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)nwid,com.signedBy().toString().c_str()); - return false; // invalid signature - } - - } else { - - SharedPtr signer(RR->topology->getPeer(com.signedBy())); - - if (!signer) { - // This would be rather odd, since this is our controller... could happen - // if we get packets before we've gotten config. - RR->sw->requestWhois(com.signedBy()); - return false; // signer unknown - } - - if (!com.verify(signer->identity())) { - TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)nwid,com.signedBy().toString().c_str()); - return false; // invalid signature - } - } - - // If we made it past all those checks, add or update cert in our cert info store - { - Mutex::Lock _l(_networkComs_m); - _networkComs.set(nwid,_NetworkCom(RR->node->now(),com)); - } - - return true; -} - -bool Peer::needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime) -{ - Mutex::Lock _l(_networkComs_m); - uint64_t &lastPushed = _lastPushedComs[nwid]; - const uint64_t tmp = lastPushed; - if (updateLastPushedTime) - lastPushed = now; - return ((now - tmp) >= (ZT_NETWORK_AUTOCONF_DELAY / 3)); -} - void Peer::clean(uint64_t now) { { @@ -460,24 +386,13 @@ void Peer::clean(uint64_t now) } { - Mutex::Lock _l(_networkComs_m); - { - uint64_t *k = (uint64_t *)0; - _NetworkCom *v = (_NetworkCom *)0; - Hashtable< uint64_t,_NetworkCom >::Iterator i(_networkComs); - while (i.next(k,v)) { - if ( (!RR->node->belongsToNetwork(*k)) && ((now - v->ts) >= ZT_PEER_NETWORK_COM_EXPIRATION) ) - _networkComs.erase(*k); - } - } - { - uint64_t *k = (uint64_t *)0; - uint64_t *v = (uint64_t *)0; - Hashtable< uint64_t,uint64_t >::Iterator i(_lastPushedComs); - while (i.next(k,v)) { - if ((now - *v) > (ZT_NETWORK_AUTOCONF_DELAY * 2)) - _lastPushedComs.erase(*k); - } + Mutex::Lock _l(_memberships_m); + uint64_t *nwid = (uint64_t *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(nwid,m)) { + if ((now - m->clean(now)) > ZT_MEMBERSHIP_EXPIRATION_TIME) + _memberships.erase(*nwid); } } } diff --git a/node/Tag.cpp b/node/Tag.cpp index 1ad17251..352ecde8 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -21,12 +21,13 @@ #include "Identity.hpp" #include "Topology.hpp" #include "Switch.hpp" +#include "Network.hpp" namespace ZeroTier { int Tag::verify(const RuntimeEnvironment *RR) const { - if (!_signedBy) + if ((!_signedBy)||(_signedBy != Network::controllerFor(_nwid))) return -1; const Identity id(RR->topology->getIdentity(_signedBy)); if (!id) { -- cgit v1.2.3 From 98152d974ada42e659e65590dec9a53d0a28ef54 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 11:40:38 -0700 Subject: More cleanup and removal of DeferredPackets, will do the latter in a more elegant way. --- include/ZeroTierOne.h | 21 ---------- node/Capability.cpp | 4 ++ node/Capability.hpp | 62 +++++++++++++-------------- node/DeferredPackets.cpp | 100 -------------------------------------------- node/DeferredPackets.hpp | 85 ------------------------------------- node/IncomingPacket.cpp | 20 +++------ node/IncomingPacket.hpp | 13 +----- node/Node.cpp | 24 ----------- node/Node.hpp | 9 ---- node/RuntimeEnvironment.hpp | 9 ---- node/Switch.cpp | 8 ++-- objects.mk | 1 - service/OneService.cpp | 4 -- 13 files changed, 45 insertions(+), 315 deletions(-) delete mode 100644 node/DeferredPackets.cpp delete mode 100644 node/DeferredPackets.hpp (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 9679cf64..88e83a6e 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1902,27 +1902,6 @@ void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs); */ void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); -/** - * Do things in the background until Node dies - * - * This function can be called from one or more background threads to process - * certain tasks in the background to improve foreground performance. It will - * not return until the Node is shut down. If threading is not enabled in - * this build it will return immediately and will do nothing. - * - * This is completely optional. If this is never called, all processing is - * done in the foreground in the various processXXXX() methods. - * - * This does NOT replace or eliminate the need to call the normal - * processBackgroundTasks() function in your main loop. This mechanism is - * used to offload the processing of expensive mssages onto background - * handler threads to prevent foreground performance degradation under - * high load. - * - * @param node Node instance - */ -void ZT_Node_backgroundThreadMain(ZT_Node *node); - /** * Get ZeroTier One version * diff --git a/node/Capability.cpp b/node/Capability.cpp index ee798a6c..0a736ca8 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -28,9 +28,11 @@ namespace ZeroTier { int Capability::verify(const RuntimeEnvironment *RR) const { try { + // There must be at least one entry, and sanity check for bad chain max length if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) return -1; + // Validate all entries in chain of custody Buffer<(sizeof(Capability) * 2)> tmp; this->serialize(tmp,true); for(unsigned int c=0;c<_maxCustodyChainLength;++c) { @@ -53,6 +55,8 @@ int Capability::verify(const RuntimeEnvironment *RR) const return 1; } } + + // We reached max custody chain length and everything was valid return 0; } catch ( ... ) {} return -1; diff --git a/node/Capability.hpp b/node/Capability.hpp index 48282708..d9b49121 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -114,6 +114,23 @@ public: */ inline uint64_t expiration() const { return _expiration; } + /** + * Check to see if a given address is a 'to' address in the custody chain + * + * This does not actually do certificate checking. That must be done with verify(). + * + * @param a Address to check + * @return True if address is present + */ + inline bool wasIssuedTo(const Address &a) const + { + for(unsigned int i=0;i tmp; this->serialize(tmp,true); + _custody[i].to = to; + _custody[i].from = from.address(); _custody[i].signature = from.sign(tmp.data(),tmp.size()); return true; } @@ -255,22 +272,21 @@ public: b.append(_id); b.append(_nwid); b.append(_expiration); - serializeRules(b,_rules,_ruleCount); - b.append((uint8_t)_maxCustodyChainLength); - for(unsigned int i=0;;++i) { - if ((i < _maxCustodyChainLength)&&(i < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)&&(_custody[i].to)) { - _custody[i].to.appendTo(b); - _custody[i].from.appendTo(b); - if (!forSign) { + + if (!forSign) { + for(unsigned int i=0;;++i) { + if ((i < _maxCustodyChainLength)&&(i < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)&&(_custody[i].to)) { + _custody[i].to.appendTo(b); + _custody[i].from.appendTo(b); b.append((uint8_t)1); // 1 == Ed25519 signature b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); // length of signature b.append(_custody[i].signature.data,ZT_C25519_SIGNATURE_LEN); + } else { + b.append((unsigned char)0,ZT_ADDRESS_LENGTH); // zero 'to' terminates chain + break; } - } else { - b.append((unsigned char)0,ZT_ADDRESS_LENGTH); // zero 'to' terminates chain - break; } } @@ -369,10 +385,9 @@ public: _id = b.template at(p); p += 4; _nwid = b.template at(p); p += 8; _expiration = b.template at(p); p += 8; - deserializeRules(b,p,_rules,_ruleCount,ZT_MAX_CAPABILITY_RULES); - _maxCustodyChainLength = (unsigned int)b[p++]; + if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) throw std::runtime_error("invalid max custody chain length"); for(unsigned int i;;++i) { @@ -393,25 +408,6 @@ public: return (p - startAt); } - /** - * Check to see if a given address is a 'to' address in the custody chain - * - * This does not actually do certificate checking. That must be done with verify(). - * - * @param a Address to check - * @return True if address is present - */ - inline bool wasIssuedTo(const Address &a) const - { - for(unsigned int i=0;i. - */ - -#include "Constants.hpp" -#include "DeferredPackets.hpp" -#include "IncomingPacket.hpp" -#include "RuntimeEnvironment.hpp" -#include "Node.hpp" - -namespace ZeroTier { - -DeferredPackets::DeferredPackets(const RuntimeEnvironment *renv) : - RR(renv), - _waiting(0), - _die(false) -{ -} - -DeferredPackets::~DeferredPackets() -{ - _q_m.lock(); - _die = true; - _q_m.unlock(); - - for(;;) { - _q_s.post(); - - _q_m.lock(); - if (_waiting <= 0) { - _q_m.unlock(); - break; - } else { - _q_m.unlock(); - } - } -} - -bool DeferredPackets::enqueue(IncomingPacket *pkt) -{ - { - Mutex::Lock _l(_q_m); - if (_q.size() >= ZT_DEFFEREDPACKETS_MAX) - return false; - _q.push_back(*pkt); - } - _q_s.post(); - return true; -} - -int DeferredPackets::process() -{ - std::list pkt; - - _q_m.lock(); - - if (_die) { - _q_m.unlock(); - return -1; - } - - while (_q.empty()) { - ++_waiting; - _q_m.unlock(); - _q_s.wait(); - _q_m.lock(); - --_waiting; - if (_die) { - _q_m.unlock(); - return -1; - } - } - - // Move item from _q list to a dummy list here to avoid copying packet - pkt.splice(pkt.end(),_q,_q.begin()); - - _q_m.unlock(); - - try { - pkt.front().tryDecode(RR,true); - } catch ( ... ) {} // drop invalids - - return 1; -} - -} // namespace ZeroTier diff --git a/node/DeferredPackets.hpp b/node/DeferredPackets.hpp deleted file mode 100644 index a9855396..00000000 --- a/node/DeferredPackets.hpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef ZT_DEFERREDPACKETS_HPP -#define ZT_DEFERREDPACKETS_HPP - -#include - -#include "Constants.hpp" -#include "SharedPtr.hpp" -#include "Mutex.hpp" -#include "DeferredPackets.hpp" -#include "BinarySemaphore.hpp" - -/** - * Maximum number of deferred packets - */ -#define ZT_DEFFEREDPACKETS_MAX 256 - -namespace ZeroTier { - -class IncomingPacket; -class RuntimeEnvironment; - -/** - * Deferred packets - * - * IncomingPacket can defer its decoding this way by enqueueing itself here. - * When this is done, deferredDecode() is called later. This is done for - * operations that may be expensive to allow them to potentially be handled - * in the background or rate limited to maintain quality of service for more - * routine operations. - */ -class DeferredPackets -{ -public: - DeferredPackets(const RuntimeEnvironment *renv); - ~DeferredPackets(); - - /** - * Enqueue a packet - * - * @param pkt Packet to process later (possibly in the background) - * @return False if queue is full - */ - bool enqueue(IncomingPacket *pkt); - - /** - * Wait for and then process a deferred packet - * - * If we are shutting down (in destructor), this returns -1 and should - * not be called again. Otherwise it returns the number of packets - * processed. - * - * @return Number processed or -1 if shutting down - */ - int process(); - -private: - std::list _q; - const RuntimeEnvironment *const RR; - volatile int _waiting; - volatile bool _die; - Mutex _q_m; - BinarySemaphore _q_s; -}; - -} // namespace ZeroTier - -#endif diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 029570f1..ca609418 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -36,7 +36,6 @@ #include "World.hpp" #include "Cluster.hpp" #include "Node.hpp" -#include "DeferredPackets.hpp" #include "Filter.hpp" #include "CertificateOfMembership.hpp" #include "Capability.hpp" @@ -44,7 +43,7 @@ namespace ZeroTier { -bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,bool deferred) +bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) { const Address sourceAddress(source()); @@ -64,18 +63,11 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,bool deferred) return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { - // Unencrypted HELLOs require some potentially expensive verification, so - // do this in the background if background processing is enabled. - if ((RR->dpEnabled > 0)&&(!deferred)) { - RR->dp->enqueue(this); - return true; // 'handled' via deferring to background thread(s) - } else { - // A null pointer for peer to _doHELLO() tells it to run its own - // special internal authentication logic. This is done for unencrypted - // HELLOs to learn new identities, etc. - SharedPtr tmp; - return _doHELLO(RR,tmp); - } + // A null pointer for peer to _doHELLO() tells it to run its own + // special internal authentication logic. This is done for unencrypted + // HELLOs to learn new identities, etc. + SharedPtr tmp; + return _doHELLO(RR,tmp); } SharedPtr peer(RR->topology->getPeer(sourceAddress)); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index bfb30a5e..558dfaa2 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -118,21 +118,12 @@ public: * about whether the packet was valid. A rejection is 'complete.' * * Once true is returned, this must not be called again. The packet's state - * may no longer be valid. The only exception is deferred decoding. In this - * case true is returned to indicate to the normal decode path that it is - * finished with the packet. The packet will have added itself to the - * deferred queue and will expect tryDecode() to be called one more time - * with deferred set to true. - * - * Deferred decoding is performed by DeferredPackets.cpp and should not be - * done elsewhere. Under deferred decoding packets only get one shot and - * so the return value of tryDecode() is ignored. + * may no longer be valid. * * @param RR Runtime environment - * @param deferred If true, this is a deferred decode and the return is ignored * @return True if decoding and processing is complete, false if caller should try again */ - bool tryDecode(const RuntimeEnvironment *RR,bool deferred); + bool tryDecode(const RuntimeEnvironment *RR); /** * @return Time of packet receipt / start of decode diff --git a/node/Node.cpp b/node/Node.cpp index e5d04e31..f04559db 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -37,7 +37,6 @@ #include "Identity.hpp" #include "SelfAwareness.hpp" #include "Cluster.hpp" -#include "DeferredPackets.hpp" const struct sockaddr_storage ZT_SOCKADDR_NULL = {0}; @@ -108,9 +107,7 @@ Node::Node( RR->mc = new Multicaster(RR); RR->topology = new Topology(RR); RR->sa = new SelfAwareness(RR); - RR->dp = new DeferredPackets(RR); } catch ( ... ) { - delete RR->dp; delete RR->sa; delete RR->topology; delete RR->mc; @@ -127,8 +124,6 @@ Node::~Node() _networks.clear(); // ensure that networks are destroyed before shutdow - RR->dpEnabled = 0; - delete RR->dp; delete RR->sa; delete RR->topology; delete RR->mc; @@ -621,18 +616,6 @@ void Node::clusterStatus(ZT_ClusterStatus *cs) memset(cs,0,sizeof(ZT_ClusterStatus)); } -void Node::backgroundThreadMain() -{ - ++RR->dpEnabled; - for(;;) { - try { - if (RR->dp->process() < 0) - break; - } catch ( ... ) {} // sanity check -- should not throw - } - --RR->dpEnabled; -} - /****************************************************************************/ /* Node methods used only within node/ */ /****************************************************************************/ @@ -1009,13 +992,6 @@ void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networ } catch ( ... ) {} } -void ZT_Node_backgroundThreadMain(ZT_Node *node) -{ - try { - reinterpret_cast(node)->backgroundThreadMain(); - } catch ( ... ) {} -} - void ZT_version(int *major,int *minor,int *revision) { if (major) *major = ZEROTIER_ONE_VERSION_MAJOR; diff --git a/node/Node.hpp b/node/Node.hpp index 0a39d1ee..98c4fd7c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -117,18 +117,9 @@ public: void clusterRemoveMember(unsigned int memberId); void clusterHandleIncomingMessage(const void *msg,unsigned int len); void clusterStatus(ZT_ClusterStatus *cs); - void backgroundThreadMain(); // Internal functions ------------------------------------------------------ - /** - * Convenience threadMain() for easy background thread launch - * - * This allows background threads to be launched with Thread::start - * that will run against this node. - */ - inline void threadMain() throw() { this->backgroundThreadMain(); } - /** * @return Time as of last call to run() */ diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 1f527733..7ba1c989 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -35,7 +35,6 @@ class Multicaster; class NetworkController; class SelfAwareness; class Cluster; -class DeferredPackets; /** * Holds global state for an instance of ZeroTier::Node @@ -51,11 +50,9 @@ public: ,mc((Multicaster *)0) ,topology((Topology *)0) ,sa((SelfAwareness *)0) - ,dp((DeferredPackets *)0) #ifdef ZT_ENABLE_CLUSTER ,cluster((Cluster *)0) #endif - ,dpEnabled(0) { } @@ -82,15 +79,9 @@ public: Multicaster *mc; Topology *topology; SelfAwareness *sa; - DeferredPackets *dp; - #ifdef ZT_ENABLE_CLUSTER Cluster *cluster; #endif - - // This is set to >0 if background threads are waiting on deferred - // packets, otherwise 'dp' should not be used. - volatile int dpEnabled; }; } // namespace ZeroTier diff --git a/node/Switch.cpp b/node/Switch.cpp index 41756aa9..33b08429 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -165,7 +165,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from for(unsigned int f=1;ffrag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); - if (rq->frag0.tryDecode(RR,false)) { + if (rq->frag0.tryDecode(RR)) { rq->timestamp = 0; // packet decoded, free entry } else { rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something @@ -264,7 +264,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from for(unsigned int f=1;ftotalFragments;++f) rq->frag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); - if (rq->frag0.tryDecode(RR,false)) { + if (rq->frag0.tryDecode(RR)) { rq->timestamp = 0; // packet decoded, free entry } else { rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something @@ -277,7 +277,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } else { // Packet is unfragmented, so just process it IncomingPacket packet(data,len,localAddr,fromAddr,now); - if (!packet.tryDecode(RR,false)) { + if (!packet.tryDecode(RR)) { Mutex::Lock _l(_rxQueue_m); RXQueueEntry *rq = &(_rxQueue[ZT_RX_QUEUE_SIZE - 1]); unsigned long i = ZT_RX_QUEUE_SIZE - 1; @@ -705,7 +705,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) while (i) { RXQueueEntry *rq = &(_rxQueue[--i]); if ((rq->timestamp)&&(rq->complete)) { - if (rq->frag0.tryDecode(RR,false)) + if (rq->frag0.tryDecode(RR)) rq->timestamp = 0; } } diff --git a/objects.mk b/objects.mk index 18e330b3..99cf1a72 100644 --- a/objects.mk +++ b/objects.mk @@ -3,7 +3,6 @@ OBJS=\ node/Capability.o \ node/CertificateOfMembership.o \ node/Cluster.o \ - node/DeferredPackets.o \ node/Filter.o \ node/Identity.o \ node/IncomingPacket.o \ diff --git a/service/OneService.cpp b/service/OneService.cpp index 460eb1c9..0c9b0b8e 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -864,10 +864,6 @@ public: } } - // Start two background threads to handle expensive ops out of line - Thread::start(_node); - Thread::start(_node); - _nextBackgroundTaskDeadline = 0; uint64_t clockShouldBe = OSUtils::now(); _lastRestart = clockShouldBe; -- cgit v1.2.3 From 331382cf2f3a3da9c5ec6821f9d63d3f3452202a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 12:14:13 -0700 Subject: More cleanup and a tiny federation prep item. --- node/IncomingPacket.cpp | 15 ++++++++++----- node/Network.cpp | 6 +----- 2 files changed, 11 insertions(+), 10 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ca609418..aea110d5 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -465,12 +465,13 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr { try { if (payloadLength() == ZT_ADDRESS_LENGTH) { - Identity queried(RR->topology->getIdentity(Address(payload(),ZT_ADDRESS_LENGTH))); - if (queried) { + const Address addr(payload(),ZT_ADDRESS_LENGTH); + const Identity id(RR->topology->getIdentity(addr)); + if (id) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_WHOIS); outp.append(packetId()); - queried.serialize(outp,false); + id.serialize(outp,false); outp.armor(peer->key(),true); RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } else { @@ -478,6 +479,10 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr if (RR->cluster) RR->cluster->sendDistributedQuery(*this); #endif + if (!RR->topology->amRoot()) { + RR->sw->requestWhois(addr); + return false; // packet parse will be attempted again if we get a reply from upstream + } } } else { TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str()); @@ -492,7 +497,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - if (RR->topology->isUpstream(peer->identity())) { + if (RR->topology->isUpstream(peer->identity())) { // only upstream peers can tell us to rendezvous, otherwise this opens a potential amplification attack vector const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); const SharedPtr withPeer(RR->topology->getPeer(with)); if (withPeer) { @@ -501,7 +506,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP); - InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); + const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) RR->sw->rendezvous(withPeer,_localAddress,atAddr); diff --git a/node/Network.cpp b/node/Network.cpp index d9ad7838..485a598b 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -50,10 +50,6 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : { char confn[128],mcdbn[128]; Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id); - Utils::snprintf(mcdbn,sizeof(mcdbn),"networks.d/%.16llx.mcerts",_id); - - // These files are no longer used, so clean them. - RR->node->dataStoreDelete(mcdbn); if (_id == ZT_TEST_NETWORK_ID) { applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address())); @@ -144,7 +140,7 @@ bool Network::tryAnnounceMulticastGroupsTo(const SharedPtr &peer) if ( (_isAllowed(peer)) || (peer->address() == this->controller()) || - (RR->topology->isRoot(peer->identity())) + (RR->topology->isUpstream(peer->identity())) ) { _announceMulticastGroupsTo(peer,_allMulticastGroups()); return true; -- cgit v1.2.3 From 8a7753cfe3824ad378e779140fdd99f5c2873642 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 12:35:25 -0700 Subject: Filter cleanup, prep for filter integration in a few places. --- node/Filter.cpp | 11 +++---- node/Filter.hpp | 11 +++---- node/OutboundMulticast.cpp | 71 ++++++++++++++++------------------------------ node/OutboundMulticast.hpp | 10 +++---- 4 files changed, 40 insertions(+), 63 deletions(-) (limited to 'node') diff --git a/node/Filter.cpp b/node/Filter.cpp index 2980149b..286a0144 100644 --- a/node/Filter.cpp +++ b/node/Filter.cpp @@ -66,7 +66,8 @@ bool Filter::run( const unsigned int vlanId, const ZT_VirtualNetworkRule *rules, const unsigned int ruleCount, - const Tag *tags, + const uint32_t *tagKeys, + const uint32_t *tagValues, const unsigned int tagCount, Address &sendCopyOfPacketTo) { @@ -248,13 +249,13 @@ bool Filter::run( case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: for(unsigned int i=0;i= rules[rn].v.tag.value[0])&&(tags[i].value() <= rules[rn].v.tag.value[1])); + thisRuleMatches = (uint8_t)((tagValues[i] >= rules[rn].v.tag.value[0])&&(tagValues[i] <= rules[rn].v.tag.value[1])); } else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL) { - thisRuleMatches = (uint8_t)((tags[i].value() & rules[rn].v.tag.value[0]) == rules[rn].v.tag.value[0]); + thisRuleMatches = (uint8_t)((tagValues[i] & rules[rn].v.tag.value[0]) == rules[rn].v.tag.value[0]); } else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY) { - thisRuleMatches = (uint8_t)((tags[i].value() & rules[rn].v.tag.value[0]) != 0); + thisRuleMatches = (uint8_t)((tagValues[i] & rules[rn].v.tag.value[0]) != 0); } break; } diff --git a/node/Filter.hpp b/node/Filter.hpp index 06aae55f..0f02bf60 100644 --- a/node/Filter.hpp +++ b/node/Filter.hpp @@ -27,12 +27,11 @@ #include "../include/ZeroTierOne.h" #include "Address.hpp" #include "MAC.hpp" -#include "Tag.hpp" namespace ZeroTier { /** - * Network packet filter for rules engine + * A simple network packet filter with VL1, L2, and basic L3 rule support (and tags!) */ class Filter { @@ -55,8 +54,9 @@ public: * @param vlanId 16-bit VLAN ID * @param rules Pointer to array of rules * @param ruleCount Number of rules - * @param tags Tags associated with this node on this network - * @param tagCount Number of tags + * @param tagKeys Tag keys for tags that may be relevant + * @param tagValues Tag values for tags that may be relevant + * @param tagCount Size of tagKeys[] and tagValues[] * @param sendCopyOfPacketTo Result parameter: if non-NULL send a copy of this packet to another node * @return True if packet should be accepted for send or receive */ @@ -73,7 +73,8 @@ public: const unsigned int vlanId, const ZT_VirtualNetworkRule *rules, const unsigned int ruleCount, - const Tag *tags, + const uint32_t *tagKeys, + const uint32_t *tagValues, const unsigned int tagCount, Address &sendCopyOfPacketTo); }; diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index eea1132c..344e0321 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -21,8 +21,9 @@ #include "OutboundMulticast.hpp" #include "Switch.hpp" #include "Network.hpp" -#include "CertificateOfMembership.hpp" #include "Node.hpp" +#include "Peer.hpp" +#include "Topology.hpp" namespace ZeroTier { @@ -30,7 +31,6 @@ void OutboundMulticast::init( const RuntimeEnvironment *RR, uint64_t timestamp, uint64_t nwid, - const CertificateOfMembership *com, unsigned int limit, unsigned int gatherLimit, const MAC &src, @@ -48,7 +48,7 @@ void OutboundMulticast::init( if (src) flags |= 0x04; /* - TRACE(">>MC %.16llx INIT %.16llx/%s limit %u gatherLimit %u from %s to %s length %u com==%d", + TRACE(">>MC %.16llx INIT %.16llx/%s limit %u gatherLimit %u from %s to %s length %u", (unsigned long long)this, nwid, dest.toString().c_str(), @@ -56,58 +56,35 @@ void OutboundMulticast::init( gatherLimit, (src) ? src.toString().c_str() : MAC(RR->identity.address(),nwid).toString().c_str(), dest.toString().c_str(), - len, - (com) ? 1 : 0); + len); */ - _packetNoCom.setSource(RR->identity.address()); - _packetNoCom.setVerb(Packet::VERB_MULTICAST_FRAME); - _packetNoCom.append((uint64_t)nwid); - _packetNoCom.append(flags); - if (gatherLimit) _packetNoCom.append((uint32_t)gatherLimit); - if (src) src.appendTo(_packetNoCom); - dest.mac().appendTo(_packetNoCom); - _packetNoCom.append((uint32_t)dest.adi()); - _packetNoCom.append((uint16_t)etherType); - _packetNoCom.append(payload,len); - _packetNoCom.compress(); - - if (com) { - _haveCom = true; - flags |= 0x01; - - _packetWithCom.setSource(RR->identity.address()); - _packetWithCom.setVerb(Packet::VERB_MULTICAST_FRAME); - _packetWithCom.append((uint64_t)nwid); - _packetWithCom.append(flags); - com->serialize(_packetWithCom); - if (gatherLimit) _packetWithCom.append((uint32_t)gatherLimit); - if (src) src.appendTo(_packetWithCom); - dest.mac().appendTo(_packetWithCom); - _packetWithCom.append((uint32_t)dest.adi()); - _packetWithCom.append((uint16_t)etherType); - _packetWithCom.append(payload,len); - _packetWithCom.compress(); - } else _haveCom = false; + _packet.setSource(RR->identity.address()); + _packet.setVerb(Packet::VERB_MULTICAST_FRAME); + _packet.append((uint64_t)nwid); + _packet.append(flags); + if (gatherLimit) _packet.append((uint32_t)gatherLimit); + if (src) src.appendTo(_packet); + dest.mac().appendTo(_packet); + _packet.append((uint32_t)dest.adi()); + _packet.append((uint16_t)etherType); + _packet.append(payload,len); + _packet.compress(); } void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) { - if (_haveCom) { - SharedPtr peer(RR->topology->getPeer(toAddr)); - if ( (!peer) || (peer->needsOurNetworkMembershipCertificate(_nwid,RR->node->now(),true)) ) { - //TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str()); - _packetWithCom.newInitializationVector(); - _packetWithCom.setDestination(toAddr); - RR->sw->send(_packetWithCom,true,_nwid); - return; - } + // TODO: apply Filter + + SharedPtr peer(RR->topology->getPeer(toAddr)); + if (peer) { + // TODO: push creds if needed } - //TRACE(">>MC %.16llx -> %s (without COM)",(unsigned long long)this,toAddr.toString().c_str()); - _packetNoCom.newInitializationVector(); - _packetNoCom.setDestination(toAddr); - RR->sw->send(_packetNoCom,true,_nwid); + //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); + _packet.newInitializationVector(); + _packet.setDestination(toAddr); + RR->sw->send(_packet,true,_nwid); } } // namespace ZeroTier diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 3818172e..7d1dff80 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -56,7 +56,6 @@ public: * @param RR Runtime environment * @param timestamp Creation time * @param nwid Network ID - * @param com Certificate of membership or NULL if none available * @param limit Multicast limit for desired number of packets to send * @param gatherLimit Number to lazily/implicitly gather with this frame or 0 for none * @param src Source MAC address of frame or NULL to imply compute from sender ZT address @@ -70,7 +69,6 @@ public: const RuntimeEnvironment *RR, uint64_t timestamp, uint64_t nwid, - const CertificateOfMembership *com, unsigned int limit, unsigned int gatherLimit, const MAC &src, @@ -127,17 +125,17 @@ public: if (std::find(_alreadySentTo.begin(),_alreadySentTo.end(),toAddr) == _alreadySentTo.end()) { sendAndLog(RR,toAddr); return true; - } else return false; + } else { + return false; + } } private: uint64_t _timestamp; uint64_t _nwid; unsigned int _limit; - Packet _packetNoCom; - Packet _packetWithCom; + Packet _packet; std::vector
_alreadySentTo; - bool _haveCom; }; } // namespace ZeroTier -- cgit v1.2.3 From 37d139177dfdc6a0cf44f964a315184fca3fc3bd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 13:01:14 -0700 Subject: Integrate Filter into OutboundMulticast properly. --- node/Filter.hpp | 2 -- node/Network.hpp | 56 ++++++++++++++++++++++++++++++++++++++++++++++ node/OutboundMulticast.cpp | 26 ++++++++++++--------- node/OutboundMulticast.hpp | 5 +++++ 4 files changed, 76 insertions(+), 13 deletions(-) (limited to 'node') diff --git a/node/Filter.hpp b/node/Filter.hpp index 0f02bf60..a4643352 100644 --- a/node/Filter.hpp +++ b/node/Filter.hpp @@ -21,8 +21,6 @@ #include -#include - #include "Constants.hpp" #include "../include/ZeroTierOne.h" #include "Address.hpp" diff --git a/node/Network.hpp b/node/Network.hpp index 17eed4bd..10714a7a 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -77,6 +77,62 @@ public: ~Network(); + /** + * Apply filters to an outgoing packet + * + * This applies filters from our network config and, if that doesn't match, + * our capabilities in ascending order of capability ID. If there is a match + * certain actions may be taken such as pushing credentials to ztDest and + * sending a copy of the packet to a TEE or REDIRECT target. + * + * @param ztSource Source ZeroTier address + * @param ztDest Destination ZeroTier address + * @param macSource Ethernet layer source address + * @param macDest Ethernet layer destination address + * @param frameData Ethernet frame data + * @param frameLen Ethernet frame payload length + * @param etherType 16-bit ethernet type ID + * @param vlanId 16-bit VLAN ID + * @return True if packet should be sent to destination peer + */ + bool filterOutgoingPacket( + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId); + + /** + * Apply filters to an incoming packet + * + * This applies filters from our network config and, if that doesn't match, + * the peer's capabilities in ascending order of capability ID. If there is + * a match certain actions may be taken such as sending a copy of the packet + * to a TEE or REDIRECT target. + * + * @param ztSource Source Peer (to save an extra lookup) + * @param ztDest Destination ZeroTier address + * @param macSource Ethernet layer source address + * @param macDest Ethernet layer destination address + * @param frameData Ethernet frame data + * @param frameLen Ethernet frame payload length + * @param etherType 16-bit ethernet type ID + * @param vlanId 16-bit VLAN ID + * @return True if packet should be accepted locally + */ + bool filterIncomingPacket( + const SharedPtr &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId); + /** * @return Network ID */ diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 344e0321..11268fe2 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -41,7 +41,13 @@ void OutboundMulticast::init( { _timestamp = timestamp; _nwid = nwid; + if (src) + _macSrc = src; + else _macSrc.fromAddress(RR->identity.address(),nwid); + _macDest = dest.mac(); _limit = limit; + _frameLen = (len < ZT_MAX_MTU) ? len : ZT_MAX_MTU; + _etherType = etherType; uint8_t flags = 0; if (gatherLimit) flags |= 0x02; @@ -68,23 +74,21 @@ void OutboundMulticast::init( dest.mac().appendTo(_packet); _packet.append((uint32_t)dest.adi()); _packet.append((uint16_t)etherType); - _packet.append(payload,len); + _packet.append(payload,_frameLen); _packet.compress(); + + memcpy(_frameData,payload,_frameLen); } void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) { - // TODO: apply Filter - - SharedPtr peer(RR->topology->getPeer(toAddr)); - if (peer) { - // TODO: push creds if needed + const SharedPtr nw(RR->node->network(_nwid)); + if ((nw)&&(nw->filterOutgoingPacket(RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); + _packet.newInitializationVector(); + _packet.setDestination(toAddr); + RR->sw->send(_packet,true,_nwid); } - - //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); - _packet.newInitializationVector(); - _packet.setDestination(toAddr); - RR->sw->send(_packet,true,_nwid); } } // namespace ZeroTier diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 7d1dff80..0ded8baf 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -133,9 +133,14 @@ public: private: uint64_t _timestamp; uint64_t _nwid; + MAC _macSrc; + MAC _macDest; unsigned int _limit; + unsigned int _frameLen; + unsigned int _etherType; Packet _packet; std::vector
_alreadySentTo; + uint8_t _frameData[ZT_MAX_MTU]; }; } // namespace ZeroTier -- cgit v1.2.3 From 4d9b74b171d243abe2d2d6a0039865ece8a4a00c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Aug 2016 15:27:20 -0700 Subject: . --- include/ZeroTierOne.h | 5 +++++ node/Filter.cpp | 7 +++--- node/IncomingPacket.cpp | 60 ++++--------------------------------------------- 3 files changed, 13 insertions(+), 59 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 88e83a6e..2a70417e 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -164,6 +164,11 @@ extern "C" { */ #define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48) +/** + * Packet characteristics flag: packet direction, 1 for incoming 0 for outgoing + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_0_INBOUND 0x0000000000000001ULL + /** * A null/empty sockaddr (all zero) to signify an unspecified socket address */ diff --git a/node/Filter.cpp b/node/Filter.cpp index 286a0144..b8b0bd2a 100644 --- a/node/Filter.cpp +++ b/node/Filter.cpp @@ -239,9 +239,10 @@ bool Filter::run( thisRuleMatches = 0; } break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - // TODO: not supported yet - break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { + uint64_t cf = (receiving) ? ZT_RULE_PACKET_CHARACTERISTICS_0_INBOUND : 0ULL; + thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); + } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index aea110d5..c2df7ee2 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -36,7 +36,6 @@ #include "World.hpp" #include "Cluster.hpp" #include "Node.hpp" -#include "Filter.hpp" #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" @@ -541,23 +540,8 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr const MAC sourceMac(peer->address(),network->id()); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (Filter::run( - RR, - network->id(), - peer->address(), - RR->identity.address(), - sourceMac, - network->mac(), - frameData, - frameLen, - etherType, - 0, - network->config().rules, - network->config().ruleCount)) - { + if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0)) { RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); - } else { - TRACE("dropped FRAME from %s(%s): Filter::run() == false (will still log packet as received)",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); } } @@ -600,11 +584,6 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); - return true; - } - if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); return true; @@ -626,23 +605,9 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

id(), - peer->address(), - RR->identity.address(), - from, - to, - frameData, - frameLen, - etherType, - 0, - network->config().rules, - network->config().ruleCount)) - { + + if (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); - } else { - TRACE("dropped EXT_FRAME from %s(%s): Filter::run() == false (will still log packet as received)",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); } } @@ -916,25 +881,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (Filter::run( - RR, - network->id(), - peer->address(), - RR->identity.address(), - from, - to.mac(), - frameData, - frameLen, - etherType, - 0, - network->config().rules, - network->config().ruleCount)) - { + if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0)) { RR->node->putFrame(network->id(),network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); - } else { - TRACE("dropped MULTICAST_FRAME from %s(%s): Filter::run() == false (will still do implicit gather)",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id()); - // Note: we continue here since we still do implicit gather in this case... we just do not putFrame() if it - // fails the filter check. } } -- cgit v1.2.3 From e2f783ebbd39466bc03bf115b20064d222b91944 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 5 Aug 2016 15:02:01 -0700 Subject: . --- attic/LockingPtr.hpp | 99 +++++++++++ include/ZeroTierOne.h | 85 ++++++++-- node/Capability.hpp | 33 ++-- node/Hashtable.hpp | 4 +- node/LockingPtr.hpp | 99 ----------- node/Membership.cpp | 43 +++-- node/Membership.hpp | 56 ++++--- node/Network.cpp | 400 ++++++++++++++++++++++++++++++++++++++++++++- node/Network.hpp | 7 +- node/OutboundMulticast.cpp | 11 +- node/Packet.hpp | 2 + node/Peer.cpp | 29 +--- node/Peer.hpp | 33 ---- node/Tag.hpp | 2 +- 14 files changed, 660 insertions(+), 243 deletions(-) create mode 100644 attic/LockingPtr.hpp delete mode 100644 node/LockingPtr.hpp (limited to 'node') diff --git a/attic/LockingPtr.hpp b/attic/LockingPtr.hpp new file mode 100644 index 00000000..c373129a --- /dev/null +++ b/attic/LockingPtr.hpp @@ -0,0 +1,99 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_LOCKINGPTR_HPP +#define ZT_LOCKINGPTR_HPP + +#include "Mutex.hpp" + +namespace ZeroTier { + +/** + * A simple pointer that locks and holds a mutex until destroyed + * + * Care must be taken when using this. It's not very sophisticated and does + * not handle being copied except for the simple return use case. When it is + * copied it hands off the mutex to the copy and clears it in the original, + * meaning that the mutex is unlocked when the last LockingPtr<> in a chain + * of such handoffs is destroyed. If this chain of handoffs "forks" (more than + * one copy is made) then non-determinism may ensue. + * + * This does not delete or do anything else with the pointer. It also does not + * take care of locking the lock. That must be done beforehand. + */ +template +class LockingPtr +{ +public: + LockingPtr() : + _ptr((T *)0), + _lock((Mutex *)0) + { + } + + LockingPtr(T *obj,Mutex *lock) : + _ptr(obj), + _lock(lock) + { + } + + LockingPtr(const LockingPtr &p) : + _ptr(p._ptr), + _lock(p._lock) + { + const_cast(&p)->_lock = (Mutex *)0; + } + + ~LockingPtr() + { + if (_lock) + _lock->unlock(); + } + + inline LockingPtr &operator=(const LockingPtr &p) + { + _ptr = p._ptr; + _lock = p._lock; + const_cast(&p)->_lock = (Mutex *)0; + return *this; + } + + inline operator bool() const throw() { return (_ptr != (T *)0); } + inline T &operator*() const throw() { return *_ptr; } + inline T *operator->() const throw() { return _ptr; } + + /** + * @return Raw pointer to held object + */ + inline T *ptr() const throw() { return _ptr; } + + inline bool operator==(const LockingPtr &sp) const throw() { return (_ptr == sp._ptr); } + inline bool operator!=(const LockingPtr &sp) const throw() { return (_ptr != sp._ptr); } + inline bool operator>(const LockingPtr &sp) const throw() { return (_ptr > sp._ptr); } + inline bool operator<(const LockingPtr &sp) const throw() { return (_ptr < sp._ptr); } + inline bool operator>=(const LockingPtr &sp) const throw() { return (_ptr >= sp._ptr); } + inline bool operator<=(const LockingPtr &sp) const throw() { return (_ptr <= sp._ptr); } + +private: + T *_ptr; + Mutex *_lock; +}; + +} // namespace ZeroTier + +#endif diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 2a70417e..aa7ecc2c 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -165,9 +165,69 @@ extern "C" { #define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48) /** - * Packet characteristics flag: packet direction, 1 for incoming 0 for outgoing + * Packet characteristics flag: packet direction, 1 if inbound 0 if outbound */ -#define ZT_RULE_PACKET_CHARACTERISTICS_0_INBOUND 0x0000000000000001ULL +#define ZT_RULE_PACKET_CHARACTERISTICS_INBOUND 0x8000000000000000ULL + +/** + * Packet characteristics flag: TCP left-most reserved bit + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_RESERVED_0 0x0000000000000800ULL + +/** + * Packet characteristics flag: TCP middle reserved bit + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_RESERVED_1 0x0000000000000400ULL + +/** + * Packet characteristics flag: TCP right-most reserved bit + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_RESERVED_2 0x0000000000000200ULL + +/** + * Packet characteristics flag: TCP NS flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_NS 0x0000000000000100ULL + +/** + * Packet characteristics flag: TCP CWR flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_CWR 0x0000000000000080ULL + +/** + * Packet characteristics flag: TCP ECE flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_ECE 0x0000000000000040ULL + +/** + * Packet characteristics flag: TCP URG flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_URG 0x0000000000000020ULL + +/** + * Packet characteristics flag: TCP ACK flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_ACK 0x0000000000000010ULL + +/** + * Packet characteristics flag: TCP PSH flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_PSH 0x0000000000000008ULL + +/** + * Packet characteristics flag: TCP RST flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_RST 0x0000000000000004ULL + +/** + * Packet characteristics flag: TCP SYN flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_SYN 0x0000000000000002ULL + +/** + * Packet characteristics flag: TCP FIN flag + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_TCP_FIN 0x0000000000000001ULL /** * A null/empty sockaddr (all zero) to signify an unspecified socket address @@ -533,19 +593,24 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 49, /** - * Match a range of tag values (equality match if start==end) + * Match if local and remote tags differ by no more than value, use 0 to check for equality + */ + ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 50, + + /** + * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE = 50, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 51, /** - * Match if all bits are set in a tag value + * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL = 51, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 52, /** - * Match if any bit from a mask is set in a tag value + * Match if local and remote tags XORed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY = 52 + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 53 }; /** @@ -650,11 +715,11 @@ typedef struct uint16_t frameSize[2]; /** - * For matching tag values + * For tag-related rules */ struct { uint32_t id; - uint32_t value[2]; // only [0] is used for BITS_ALL or BITS_ANY, [0]-[1] for range + uint32_t value; } tag; } v; } ZT_VirtualNetworkRule; diff --git a/node/Capability.hpp b/node/Capability.hpp index d9b49121..53457d4d 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -52,13 +52,13 @@ class RuntimeEnvironment; * On the receiving side the receiver does the following for each packet: * * (1) Evaluates the capabilities of the sender (that the sender has - * presented) to determine if the sender was allowed to send this. + * presented) to determine if it should received this packet. * (2) Evaluates its own capabilities to determine if it should receive - * and process this packet. + * this packet. * (3) If both check out, it receives the packet. * * Note that rules in capabilities can do other things as well such as TEE - * or REDIRECT packets. See Filter and ZT_VirtualNetworkRule. + * or REDIRECT packets. See filter code and ZT_VirtualNetworkRule. */ class Capability { @@ -248,17 +248,13 @@ public: b.append((uint16_t)rules[i].v.frameSize[0]); b.append((uint16_t)rules[i].v.frameSize[1]); break; - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE: - b.append((uint8_t)12); - b.append((uint32_t)rules[i].v.tag.id); - b.append((uint32_t)rules[i].v.tag.value[0]); - b.append((uint32_t)rules[i].v.tag.value[1]); - break; - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: b.append((uint8_t)8); b.append((uint32_t)rules[i].v.tag.id); - b.append((uint32_t)rules[i].v.tag.value[0]); + b.append((uint32_t)rules[i].v.tag.value); break; } } @@ -360,15 +356,12 @@ public: rules[i].v.frameSize[0] = b.template at(p); rules[i].v.frameSize[0] = b.template at(p + 2); break; - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE: - rules[i].v.tag.id = b.template at(p); - rules[i].v.tag.value[0] = b.template at(p + 4); - rules[i].v.tag.value[1] = b.template at(p + 8); - break; - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: rules[i].v.tag.id = b.template at(p); - rules[i].v.tag.value[0] = b.template at(p + 4); + rules[i].v.tag.value = b.template at(p + 4); break; } p += fieldLen; diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index f06b2230..c550191e 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -103,9 +103,9 @@ public: friend class Hashtable::Iterator; /** - * @param bc Initial capacity in buckets (default: 128, must be nonzero) + * @param bc Initial capacity in buckets (default: 64, must be nonzero) */ - Hashtable(unsigned long bc = 128) : + Hashtable(unsigned long bc = 64) : _t(reinterpret_cast<_Bucket **>(::malloc(sizeof(_Bucket *) * bc))), _bc(bc), _s(0) diff --git a/node/LockingPtr.hpp b/node/LockingPtr.hpp deleted file mode 100644 index c373129a..00000000 --- a/node/LockingPtr.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef ZT_LOCKINGPTR_HPP -#define ZT_LOCKINGPTR_HPP - -#include "Mutex.hpp" - -namespace ZeroTier { - -/** - * A simple pointer that locks and holds a mutex until destroyed - * - * Care must be taken when using this. It's not very sophisticated and does - * not handle being copied except for the simple return use case. When it is - * copied it hands off the mutex to the copy and clears it in the original, - * meaning that the mutex is unlocked when the last LockingPtr<> in a chain - * of such handoffs is destroyed. If this chain of handoffs "forks" (more than - * one copy is made) then non-determinism may ensue. - * - * This does not delete or do anything else with the pointer. It also does not - * take care of locking the lock. That must be done beforehand. - */ -template -class LockingPtr -{ -public: - LockingPtr() : - _ptr((T *)0), - _lock((Mutex *)0) - { - } - - LockingPtr(T *obj,Mutex *lock) : - _ptr(obj), - _lock(lock) - { - } - - LockingPtr(const LockingPtr &p) : - _ptr(p._ptr), - _lock(p._lock) - { - const_cast(&p)->_lock = (Mutex *)0; - } - - ~LockingPtr() - { - if (_lock) - _lock->unlock(); - } - - inline LockingPtr &operator=(const LockingPtr &p) - { - _ptr = p._ptr; - _lock = p._lock; - const_cast(&p)->_lock = (Mutex *)0; - return *this; - } - - inline operator bool() const throw() { return (_ptr != (T *)0); } - inline T &operator*() const throw() { return *_ptr; } - inline T *operator->() const throw() { return _ptr; } - - /** - * @return Raw pointer to held object - */ - inline T *ptr() const throw() { return _ptr; } - - inline bool operator==(const LockingPtr &sp) const throw() { return (_ptr == sp._ptr); } - inline bool operator!=(const LockingPtr &sp) const throw() { return (_ptr != sp._ptr); } - inline bool operator>(const LockingPtr &sp) const throw() { return (_ptr > sp._ptr); } - inline bool operator<(const LockingPtr &sp) const throw() { return (_ptr < sp._ptr); } - inline bool operator>=(const LockingPtr &sp) const throw() { return (_ptr >= sp._ptr); } - inline bool operator<=(const LockingPtr &sp) const throw() { return (_ptr <= sp._ptr); } - -private: - T *_ptr; - Mutex *_lock; -}; - -} // namespace ZeroTier - -#endif diff --git a/node/Membership.cpp b/node/Membership.cpp index a1a8eb8a..37b2c16d 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -28,49 +28,44 @@ namespace ZeroTier { -bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) +bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount) { try { Buffer capsAndTags; - capsAndTags.addSize(2); unsigned int appendedCaps = 0; - for(unsigned int i=0;iid()); if ((now - cs->lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { - if ((capsAndTags.size() + sizeof(Capability)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) - break; - const Capability *c = nconf.capability(capIds[i]); - if (c) { - c->serialize(capsAndTags); - ++appendedCaps; - cs->lastPushed = now; - } + cap->serialize(capsAndTags); + cs->lastPushed = now; + ++appendedCaps; } + capsAndTags.setAt(0,(uint16_t)appendedCaps); + } else { + capsAndTags.append((uint16_t)0); } - capsAndTags.setAt(0,(uint16_t)appendedCaps); + unsigned int appendedTags = 0; const unsigned int tagCountPos = capsAndTags.size(); capsAndTags.addSize(2); - unsigned int appendedTags = 0; for(unsigned int i=0;iid()); if ((now - ts->lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { if ((capsAndTags.size() + sizeof(Tag)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) break; - const Tag *t = nconf.tag(tagIds[i]); - if (t) { - t->serialize(capsAndTags); - ++appendedTags; - ts->lastPushed = now; - } + tags[i]->serialize(capsAndTags); + ts->lastPushed = now; + ++appendedTags; } } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - if (((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)||(appendedCaps)||(appendedTags)) { - Packet outp(peer.address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - nconf.com.serialize(outp); + if ( ((com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)) || (appendedCaps) || (appendedTags) ) { + Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + if (com) + com.serialize(outp); outp.append((uint8_t)0x00); outp.append(capsAndTags.data(),capsAndTags.size()); outp.compress(); diff --git a/node/Membership.hpp b/node/Membership.hpp index 0e72b7b1..3db25488 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -40,7 +40,6 @@ namespace ZeroTier { -class Peer; class RuntimeEnvironment; /** @@ -54,18 +53,18 @@ private: struct TState { TState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed this tag to this peer + // Last time we pushed our tag to this peer (our tag with the same ID) uint64_t lastPushed; // Last time we received this tag from this peer uint64_t lastReceived; - // Tag from peer + // Tag from peer (remote tag) Tag tag; }; struct CState { CState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed this capability to this peer + // Last time we pushed our capability to this peer (our capability with this ID) uint64_t lastPushed; // Last time we received this capability from this peer uint64_t lastReceived; @@ -90,29 +89,14 @@ public: * * @param RR Runtime environment * @param now Current time - * @param peer Peer that "owns" this membership - * @param nconf Network configuration - * @param capIds Capability IDs that this peer might need - * @param capCount Number of capability IDs - * @param tagIds Tag IDs that this peer might need + * @param peerAddress Address of member peer + * @param com Network certificate of membership (if any) + * @param cap Capability to send or 0 if none + * @param tags Tags that this peer might need * @param tagCount Number of tag IDs * @return True if we pushed something */ - bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount); - - /** - * Send COM if needed - * - * @param RR Runtime environment - * @param now Current time - * @param peer Peer that "owns" this membership - * @param nconf Network configuration - * @return True if we pushed something - */ - inline bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Peer &peer,const NetworkConfig &nconf) - { - return sendCredentialsIfNeeded(RR,now,peer,nconf,(const uint32_t *)0,0,(const uint32_t *)0,0); - } + bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount); /** * @return This peer's COM if they have sent one @@ -130,6 +114,30 @@ public: return ((t) ? (((t->lastReceived != 0)&&(t->tag.expiration() < nconf.timestamp)) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); } + /** + * @param nconf Network configuration + * @param ids Array to store IDs into + * @param values Array to store values into + * @param maxTags Capacity of ids[] and values[] + * @return Number of tags added to arrays + */ + inline unsigned int getAllTags(const NetworkConfig &nconf,uint32_t *ids,uint32_t *values,unsigned int maxTags) const + { + unsigned int n = 0; + uint32_t *id = (uint32_t *)0; + TState *ts = (TState *)0; + Hashtable::Iterator i(const_cast(this)->_tags); + while (i.next(id,ts)) { + if ((ts->lastReceived)&&(ts->tag.expiration() < nconf.timestamp)) { + if (n >= maxTags) + return n; + ids[n] = *id; + values[n] = ts->tag.value(); + } + } + return n; + } + /** * @param nconf Network configuration * @param id Capablity ID diff --git a/node/Network.cpp b/node/Network.cpp index 485a598b..314edf5c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -22,19 +22,313 @@ #include #include "Constants.hpp" +#include "../version.h" #include "Network.hpp" #include "RuntimeEnvironment.hpp" +#include "MAC.hpp" +#include "Address.hpp" +#include "InetAddress.hpp" #include "Switch.hpp" -#include "Packet.hpp" #include "Buffer.hpp" +#include "Packet.hpp" #include "NetworkController.hpp" #include "Node.hpp" #include "Peer.hpp" -#include "../version.h" - namespace ZeroTier { +// Returns true if packet appears valid; pos and proto will be set +static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) +{ + if (frameLen < 40) + return false; + pos = 40; + proto = frameData[6]; + while (pos <= frameLen) { + switch(proto) { + case 0: // hop-by-hop options + case 43: // routing + case 60: // destination options + case 135: // mobility options + if ((pos + 8) > frameLen) + return false; // invalid! + proto = frameData[pos]; + pos += ((unsigned int)frameData[pos + 1] * 8) + 8; + break; + + //case 44: // fragment -- we currently can't parse these and they are deprecated in IPv6 anyway + //case 50: + //case 51: // IPSec ESP and AH -- we have to stop here since this is encrypted stuff + default: + return true; + } + } + return false; // overflow == invalid +} + +static bool _doZtFilter( + const RuntimeEnvironment *RR, + const uint64_t nwid, + const bool inbound, + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId, + const ZT_VirtualNetworkRule *rules, + const unsigned int ruleCount, + const Tag *localTags, + const unsigned int localTagCount, + const uint32_t *remoteTagIds, + const uint32_t *remoteTagValues, + const unsigned int remoteTagCount, + const Tag **relevantLocalTags, // pointer array must be at least [localTagCount] in size + unsigned int &relevantLocalTagCount) +{ + // For each set of rules we start by assuming that they match (since no constraints + // yields a 'match all' rule). + uint8_t thisSetMatches = 1; + + for(unsigned int rn=0;rnidentity.address(),Packet::VERB_EXT_FRAME); + outp.append(nwid); + outp.append((uint8_t)((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02)); + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,frameLen); + outp.compress(); + RR->sw->send(outp,true,nwid); + + if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { + return false; + } else { + thisSetMatches = 1; // TEE does not terminate parsing + } + } break; + + // Rules --------------------------------------------------------------- + + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + // NOT SUPPORTED YET + thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + // NOT SUPPORTED YET + thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); + thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto); + } else { + thisRuleMatches = 0; + } + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + const unsigned int headerLen = 4 * (frameData[0] & 0xf); + int p = -1; + switch(frameData[9]) { // IP protocol number + // All these start with 16-bit source and destination port in that order + case 0x06: // TCP + case 0x11: // UDP + case 0x84: // SCTP + case 0x88: // UDPLite + if (frameLen > (headerLen + 4)) { + unsigned int pos = headerLen + ((rt == ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE) ? 2 : 0); + p = (int)frameData[pos++] << 8; + p |= (int)frameData[pos]; + } + break; + } + thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + int p = -1; + switch(proto) { // IP protocol number + // All these start with 16-bit source and destination port in that order + case 0x06: // TCP + case 0x11: // UDP + case 0x84: // SCTP + case 0x88: // UDPLite + if (frameLen > (pos + 4)) { + if (rt == ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE) pos += 2; + p = (int)frameData[pos++] << 8; + p |= (int)frameData[pos]; + } + break; + } + thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + } else { + thisRuleMatches = 0; + } + } else { + thisRuleMatches = 0; + } + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { + uint64_t cf = (inbound) ? ZT_RULE_PACKET_CHARACTERISTICS_INBOUND : 0ULL; + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)&&(frameData[9] == 0x06)) { + const unsigned int headerLen = 4 * (frameData[0] & 0xf); + cf |= (uint64_t)frameData[headerLen + 13]; + cf |= (((uint64_t)(frameData[headerLen + 12] & 0x0f)) << 8); + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + if ((proto == 0x06)&&(frameLen > (pos + 14))) { + cf |= (uint64_t)frameData[pos + 13]; + cf |= (((uint64_t)(frameData[pos + 12] & 0x0f)) << 8); + } + } + } + thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); + } break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); + break; + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { + const Tag *lt = (const Tag *)0; + for(unsigned int i=0;ivalue() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); + thisRuleMatches = (uint8_t)(sameness <= rules[rn].v.tag.value); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { + thisRuleMatches = (uint8_t)((lt->value() & *rtv) <= rules[rn].v.tag.value); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) { + thisRuleMatches = (uint8_t)((lt->value() | *rtv) <= rules[rn].v.tag.value); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { + thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) <= rules[rn].v.tag.value); + } else { // sanity check, can't really happen + thisRuleMatches = 0; + } + if (thisRuleMatches) { + relevantLocalTags[relevantLocalTagCount++] = lt; + } + } + } + } break; + } + + // thisSetMatches remains true if the current rule matched... or does NOT match if not bit (0x80) is 1 + thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); + + //TRACE("[%u] %u result==%u set==%u",rn,(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); + } + + return false; +} + const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : @@ -100,6 +394,96 @@ Network::~Network() } } +bool Network::filterOutgoingPacket( + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId) +{ + uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; + uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + const Tag *relevantLocalTags[ZT_MAX_NETWORK_TAGS]; + unsigned int relevantLocalTagCount = 0; + + Mutex::Lock _l(_lock); + + Membership &m = _memberships[ztDest]; + const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); + + if (_doZtFilter( + RR, + _id, + false, + ztSource, + ztDest, + macSource, + macDest, + frameData, + frameLen, + etherType, + vlanId, + _config.rules, + _config.ruleCount, + _config.tags, + _config.tagCount, + remoteTagIds, + remoteTagValues, + remoteTagCount, + relevantLocalTags, + relevantLocalTagCount + )) { + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); + return true; + } + + for(unsigned int c=0;c<_config.capabilityCount;++c) { + relevantLocalTagCount = 0; + if (_doZtFilter( + RR, + _id, + false, + ztSource, + ztDest, + macSource, + macDest, + frameData, + frameLen, + etherType, + vlanId, + _config.capabilities[c].rules(), + _config.capabilities[c].ruleCount(), + _config.tags, + _config.tagCount, + remoteTagIds, + remoteTagValues, + remoteTagCount, + relevantLocalTags, + relevantLocalTagCount + )) { + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); + return true; + } + } + + return false; +} + +bool Network::filterIncomingPacket( + const SharedPtr &sourcePeer, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId) +{ +} + bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const { Mutex::Lock _l(_lock); @@ -267,6 +651,16 @@ void Network::clean() _multicastGroupsBehindMe.erase(*mg); } } + + { + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) { + if ((now - m->clean(now)) > ZT_MEMBERSHIP_EXPIRATION_TIME) + _memberships.erase(*a); + } + } } void Network::learnBridgeRoute(const MAC &mac,const Address &addr) diff --git a/node/Network.hpp b/node/Network.hpp index 10714a7a..a8eb3156 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -40,6 +40,7 @@ #include "MAC.hpp" #include "Dictionary.hpp" #include "Multicaster.hpp" +#include "Membership.hpp" #include "NetworkConfig.hpp" #include "CertificateOfMembership.hpp" @@ -113,7 +114,7 @@ public: * a match certain actions may be taken such as sending a copy of the packet * to a TEE or REDIRECT target. * - * @param ztSource Source Peer (to save an extra lookup) + * @param sourcePeer Source Peer * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address * @param macDest Ethernet layer destination address @@ -124,7 +125,7 @@ public: * @return True if packet should be accepted locally */ bool filterIncomingPacket( - const SharedPtr &ztSource, + const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, const MAC &macDest, @@ -387,6 +388,8 @@ private: } _netconfFailure; volatile int _portError; // return value from port config callback + Hashtable _memberships; + Mutex _lock; AtomicCounter __refCount; diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 11268fe2..a5856164 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -39,19 +39,22 @@ void OutboundMulticast::init( const void *payload, unsigned int len) { + uint8_t flags = 0; + _timestamp = timestamp; _nwid = nwid; - if (src) + if (src) { _macSrc = src; - else _macSrc.fromAddress(RR->identity.address(),nwid); + flags |= 0x04; + } else { + _macSrc.fromAddress(RR->identity.address(),nwid); + } _macDest = dest.mac(); _limit = limit; _frameLen = (len < ZT_MAX_MTU) ? len : ZT_MAX_MTU; _etherType = etherType; - uint8_t flags = 0; if (gatherLimit) flags |= 0x02; - if (src) flags |= 0x04; /* TRACE(">>MC %.16llx INIT %.16llx/%s limit %u gatherLimit %u from %s to %s length %u", diff --git a/node/Packet.hpp b/node/Packet.hpp index 977dc1bc..6789580e 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -660,6 +660,8 @@ public: * * Flags: * 0x01 - Certificate of network membership attached (DEPRECATED) + * 0x02 - Packet is a TEE'd packet + * 0x04 - Packet is a REDIRECT'ed packet * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we diff --git a/node/Peer.cpp b/node/Peer.cpp index a994c4b2..ba47a0be 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -373,28 +373,15 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) void Peer::clean(uint64_t now) { - { - unsigned int np = _numPaths; - unsigned int x = 0; - unsigned int y = 0; - while (x < np) { - if (_paths[x].active(now)) - _paths[y++] = _paths[x]; - ++x; - } - _numPaths = y; - } - - { - Mutex::Lock _l(_memberships_m); - uint64_t *nwid = (uint64_t *)0; - Membership *m = (Membership *)0; - Hashtable::Iterator i(_memberships); - while (i.next(nwid,m)) { - if ((now - m->clean(now)) > ZT_MEMBERSHIP_EXPIRATION_TIME) - _memberships.erase(*nwid); - } + unsigned int np = _numPaths; + unsigned int x = 0; + unsigned int y = 0; + while (x < np) { + if (_paths[x].active(now)) + _paths[y++] = _paths[x]; + ++x; } + _numPaths = y; } void Peer::_doDeadPathDetection(Path &p,const uint64_t now) diff --git a/node/Peer.hpp b/node/Peer.hpp index 8b50f429..d8c44ebe 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -40,10 +40,8 @@ #include "SharedPtr.hpp" #include "AtomicCounter.hpp" #include "Hashtable.hpp" -#include "Membership.hpp" #include "Mutex.hpp" #include "NonCopyable.hpp" -#include "LockingPtr.hpp" namespace ZeroTier { @@ -386,34 +384,6 @@ public: return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT); } - /** - * Get the membership record for this network, possibly creating if missing - * - * @param networkId Network ID - * @param createIfMissing If true, create a Membership record if there isn't one - * @return Single-scope locking pointer (see LockingPtr.hpp) to Membership or NULL if not found and createIfMissing is false - */ - inline LockingPtr membership(const uint64_t networkId,bool createIfMissing) - { - _memberships_m.lock(); - try { - if (createIfMissing) { - return LockingPtr(&(_memberships[networkId]),&_memberships_m); - } else { - Membership *m = _memberships.get(networkId); - if (m) { - return LockingPtr(m,&_memberships_m); - } else { - _memberships_m.unlock(); - return LockingPtr(); - } - } - } catch ( ... ) { - _memberships_m.unlock(); - throw; - } - } - /** * Find a common set of addresses by which two peers can link, if any * @@ -460,9 +430,6 @@ private: unsigned int _latency; unsigned int _directPathPushCutoffCount; - Hashtable _memberships; - Mutex _memberships_m; - AtomicCounter __refCount; }; diff --git a/node/Tag.hpp b/node/Tag.hpp index dcf2eb20..a9f6f57e 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -79,7 +79,7 @@ public: inline uint64_t networkId() const { return _nwid; } inline uint64_t expiration() const { return _expiration; } inline uint32_t id() const { return _id; } - inline uint32_t value() const { return _value; } + inline const uint32_t &value() const { return _value; } inline const Address &issuedTo() const { return _issuedTo; } inline const Address &signedBy() const { return _signedBy; } -- cgit v1.2.3 From 4d7f625aa10e422395f92520c645dff40ad83d3c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 5 Aug 2016 15:55:38 -0700 Subject: . --- node/Filter.cpp | 276 ---------------------------------------------------- node/Filter.hpp | 82 ---------------- node/Membership.cpp | 2 +- node/Network.cpp | 38 ++++++++ objects.mk | 1 - 5 files changed, 39 insertions(+), 360 deletions(-) delete mode 100644 node/Filter.cpp delete mode 100644 node/Filter.hpp (limited to 'node') diff --git a/node/Filter.cpp b/node/Filter.cpp deleted file mode 100644 index b8b0bd2a..00000000 --- a/node/Filter.cpp +++ /dev/null @@ -1,276 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include - -#include "Constants.hpp" -#include "Filter.hpp" -#include "InetAddress.hpp" - -// Returns true if packet appears valid; pos and proto will be set -static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) -{ - if (frameLen < 40) - return false; - pos = 40; - proto = frameData[6]; - while (pos <= frameLen) { - switch(proto) { - case 0: // hop-by-hop options - case 43: // routing - case 60: // destination options - case 135: // mobility options - if ((pos + 8) > frameLen) - return false; // invalid! - proto = frameData[pos]; - pos += ((unsigned int)frameData[pos + 1] * 8) + 8; - break; - - //case 44: // fragment -- we currently can't parse these and they are deprecated in IPv6 anyway - //case 50: - //case 51: // IPSec ESP and AH -- we have to stop here since this is encrypted stuff - default: - return true; - } - } - return false; // overflow == invalid -} - -namespace ZeroTier { - -bool Filter::run( - const uint64_t nwid, - const bool receiving, - const Address &ztSource, - const Address &ztDest, - const MAC &macSource, - const MAC &macDest, - const uint8_t *frameData, - const unsigned int frameLen, - const unsigned int etherType, - const unsigned int vlanId, - const ZT_VirtualNetworkRule *rules, - const unsigned int ruleCount, - const uint32_t *tagKeys, - const uint32_t *tagValues, - const unsigned int tagCount, - Address &sendCopyOfPacketTo) -{ - sendCopyOfPacketTo.zero(); - - // For each set of rules we start by assuming that they match (since no constraints - // yields a 'match all' rule). - uint8_t thisSetMatches = 1; - - for(unsigned int rn=0;rnidentity.address(),Packet::VERB_EXT_FRAME); - outp.append(nwid); - outp.append((unsigned char)0x00); // TODO: should maybe include COM if needed - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,frameLen); - outp.compress(); - RR->sw->send(outp,true,nwid); - */ - } - // For REDIRECT we will want to DROP at this node. For TEE we ACCEPT at this node but - // also forward it along as we just did. - return (rt != ZT_NETWORK_RULE_ACTION_REDIRECT); - } - return false; - } else { - // Otherwise start a new set, assuming that it will match - //TRACE("[%u] %u previous set did not match, starting next",rn,(unsigned int)rt); - thisSetMatches = 1; - } - continue; - - // A rule can consist of one or more MATCH criterion - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); - break; - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - // NOT SUPPORTED YET - thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - // NOT SUPPORTED YET - thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); - break; - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { - thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { - thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); - } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { - const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); - thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); - } else if (etherType == ZT_ETHERTYPE_IPV6) { - unsigned int pos = 0,proto = 0; - if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { - thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto); - } else { - thisRuleMatches = 0; - } - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - const unsigned int headerLen = 4 * (frameData[0] & 0xf); - int p = -1; - switch(frameData[9]) { // IP protocol number - // All these start with 16-bit source and destination port in that order - case 0x06: // TCP - case 0x11: // UDP - case 0x84: // SCTP - case 0x88: // UDPLite - if (frameLen > (headerLen + 4)) { - unsigned int pos = headerLen + (((unsigned int)(rt == ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE)) << 1); // headerLen or +2 for destination port - p = (int)frameData[pos++] << 8; - p |= (int)frameData[pos]; - } - break; - } - thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; - } else if (etherType == ZT_ETHERTYPE_IPV6) { - unsigned int pos = 0,proto = 0; - if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { - int p = -1; - switch(proto) { // IP protocol number - // All these start with 16-bit source and destination port in that order - case 0x06: // TCP - case 0x11: // UDP - case 0x84: // SCTP - case 0x88: // UDPLite - if (frameLen > (pos + 4)) { - p = (int)frameData[pos++] << 8; - p |= (int)frameData[pos]; - } - break; - } - thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; - } else { - thisRuleMatches = 0; - } - } else { - thisRuleMatches = 0; - } - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { - uint64_t cf = (receiving) ? ZT_RULE_PACKET_CHARACTERISTICS_0_INBOUND : 0ULL; - thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); - } break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); - break; - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE: - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL: - case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY: - for(unsigned int i=0;i= rules[rn].v.tag.value[0])&&(tagValues[i] <= rules[rn].v.tag.value[1])); - } else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL) { - thisRuleMatches = (uint8_t)((tagValues[i] & rules[rn].v.tag.value[0]) == rules[rn].v.tag.value[0]); - } else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY) { - thisRuleMatches = (uint8_t)((tagValues[i] & rules[rn].v.tag.value[0]) != 0); - } - break; - } - } - break; - } - - // thisSetMatches remains true if the current rule matched... or does NOT match if not bit (0x80) is 1 - thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); - - //TRACE("[%u] %u result==%u set==%u",rn,(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); - } - - return false; // no matches, no rules, default action is therefore DROP -} - -} // namespace ZeroTier diff --git a/node/Filter.hpp b/node/Filter.hpp deleted file mode 100644 index a4643352..00000000 --- a/node/Filter.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef ZT_FILTER_HPP -#define ZT_FILTER_HPP - -#include - -#include "Constants.hpp" -#include "../include/ZeroTierOne.h" -#include "Address.hpp" -#include "MAC.hpp" - -namespace ZeroTier { - -/** - * A simple network packet filter with VL1, L2, and basic L3 rule support (and tags!) - */ -class Filter -{ -public: - /** - * Apply a list of rules to a packet - * - * This returns whether or not the packet should be accepted and may also - * take other actions for e.g. the TEE and REDIRECT targets. - * - * @param nwid ZeroTier network ID - * @param receiving True if on receiving side, false on sending side - * @param ztSource Source ZeroTier address - * @param ztDest Destination ZeroTier address - * @param macSource Ethernet layer source address - * @param macDest Ethernet layer destination address - * @param frameData Ethernet frame data - * @param frameLen Ethernet frame payload length - * @param etherType 16-bit ethernet type ID - * @param vlanId 16-bit VLAN ID - * @param rules Pointer to array of rules - * @param ruleCount Number of rules - * @param tagKeys Tag keys for tags that may be relevant - * @param tagValues Tag values for tags that may be relevant - * @param tagCount Size of tagKeys[] and tagValues[] - * @param sendCopyOfPacketTo Result parameter: if non-NULL send a copy of this packet to another node - * @return True if packet should be accepted for send or receive - */ - static bool run( - const uint64_t nwid, - const bool receiving, - const Address &ztSource, - const Address &ztDest, - const MAC &macSource, - const MAC &macDest, - const uint8_t *frameData, - const unsigned int frameLen, - const unsigned int etherType, - const unsigned int vlanId, - const ZT_VirtualNetworkRule *rules, - const unsigned int ruleCount, - const uint32_t *tagKeys, - const uint32_t *tagValues, - const unsigned int tagCount, - Address &sendCopyOfPacketTo); -}; - -} // namespace ZeroTier - -#endif diff --git a/node/Membership.cpp b/node/Membership.cpp index 37b2c16d..ec1713f0 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -24,7 +24,7 @@ #include "Packet.hpp" #include "Node.hpp" -#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 2) +#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 4) namespace ZeroTier { diff --git a/node/Network.cpp b/node/Network.cpp index 314edf5c..1c894306 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -482,6 +482,44 @@ bool Network::filterIncomingPacket( const unsigned int etherType, const unsigned int vlanId) { + uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; + uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + const Tag *relevantLocalTags[ZT_MAX_NETWORK_TAGS]; + unsigned int relevantLocalTagCount = 0; + + Mutex::Lock _l(_lock); + + Membership &m = _memberships[ztDest]; + const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); + + if (_doZtFilter( + RR, + _id, + true, + sourcePeer->address(), + ztDest, + macSource, + macDest, + frameData, + frameLen, + etherType, + vlanId, + _config.rules, + _config.ruleCount, + _config.tags, + _config.tagCount, + remoteTagIds, + remoteTagValues, + remoteTagCount, + relevantLocalTags, + relevantLocalTagCount + )) { + return true; + } + + + + return false; } bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const diff --git a/objects.mk b/objects.mk index 99cf1a72..11b03c81 100644 --- a/objects.mk +++ b/objects.mk @@ -3,7 +3,6 @@ OBJS=\ node/Capability.o \ node/CertificateOfMembership.o \ node/Cluster.o \ - node/Filter.o \ node/Identity.o \ node/IncomingPacket.o \ node/InetAddress.o \ -- cgit v1.2.3 From 8007ca56aaa2781e068ce9e3849a64b1e7bf7b8f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 Aug 2016 16:50:00 -0700 Subject: Refactor and tie-up of capabilities and tags and packet evaluation points. Some optimization is possible here but it is minor and we will make it work first. --- node/Membership.cpp | 26 +++++----- node/Membership.hpp | 80 ++++++++++++++++++++----------- node/Network.cpp | 136 +++++++++++++++++++--------------------------------- node/Network.hpp | 2 +- 4 files changed, 118 insertions(+), 126 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index ec1713f0..79b1e1bc 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -36,10 +36,10 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint unsigned int appendedCaps = 0; if (cap) { capsAndTags.addSize(2); - CState *const cs = _caps.get(cap->id()); - if ((now - cs->lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { + std::map::iterator cs(_caps.find(cap->id())); + if ((cs != _caps.end())&&((now - cs->second.lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY)) { cap->serialize(capsAndTags); - cs->lastPushed = now; + cs->second.lastPushed = now; ++appendedCaps; } capsAndTags.setAt(0,(uint16_t)appendedCaps); @@ -95,13 +95,13 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,co { if (tag.issuedTo() != RR->identity.address()) return -1; - TState *t = _tags.get(tag.networkId()); + TState *t = _tags.get(tag.id()); if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) return 0; const int vr = tag.verify(RR); if (vr == 0) { if (!t) - t = &(_tags[tag.networkId()]); + t = &(_tags[tag.id()]); t->lastReceived = now; t->tag = tag; } @@ -112,15 +112,19 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,co { if (!cap.wasIssuedTo(RR->identity.address())) return -1; - CState *c = _caps.get(cap.networkId()); - if ((c)&&(c->lastReceived != 0)&&(c->cap == cap)) + std::map::iterator c(_caps.find(cap.id())); + if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) return 0; const int vr = cap.verify(RR); if (vr == 0) { - if (!c) - c = &(_caps[cap.networkId()]); - c->lastReceived = now; - c->cap = cap; + if (c == _caps.end()) { + CState &c2 = _caps[cap.id()]; + c2.lastReceived = now; + c2.cap = cap; + } else { + c->second.lastReceived = now; + c->second.cap = cap; + } } return vr; } diff --git a/node/Membership.hpp b/node/Membership.hpp index 3db25488..664cd2ad 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -21,8 +21,7 @@ #include -#include -#include +#include #include "Constants.hpp" #include "../include/ZeroTierOne.h" @@ -43,40 +42,67 @@ namespace ZeroTier { class RuntimeEnvironment; /** - * Information related to a peer's participation on a network - * - * This structure is not thread-safe and must be locked during use. + * A container for certificates of membership and other credentials for peer participation on networks */ class Membership { private: + // Tags and related state struct TState { TState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed our tag to this peer (our tag with the same ID) + // Last time we pushed OUR tag to this peer (with this ID) uint64_t lastPushed; - // Last time we received this tag from this peer + // Last time we received THEIR tag (with this ID) uint64_t lastReceived; - // Tag from peer (remote tag) + // THEIR tag Tag tag; }; + // Credentials and related state struct CState { CState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed our capability to this peer (our capability with this ID) + // Last time we pushed OUR capability to this peer (with this ID) uint64_t lastPushed; - // Last time we received this capability from this peer + // Last time we received THEIR capability (with this ID) uint64_t lastReceived; - // Capability from peer + // THEIR capability Capability cap; }; public: + /** + * A wrapper to iterate through capabilities in ascending order of capability ID + */ + class CapabilityIterator + { + public: + CapabilityIterator(const Membership &m) : + _i(m._caps.begin()), + _e(m._caps.end()) + { + } + + inline const Capability *next() + { + while (_i != _e) { + if (_i->second.lastReceived) + return &((_i++)->second.cap); + else ++_i; + } + return (const Capability *)0; + } + + private: + std::map::const_iterator _i,_e; + }; + friend class CapabilityIterator; + Membership() : _lastPushedCom(0), _com(), - _caps(8), + _caps(), _tags(8) { } @@ -90,7 +116,7 @@ public: * @param RR Runtime environment * @param now Current time * @param peerAddress Address of member peer - * @param com Network certificate of membership (if any) + * @param com My network certificate of membership (if any) (not the one here, but ours -- in NetworkConfig) * @param cap Capability to send or 0 if none * @param tags Tags that this peer might need * @param tagCount Number of tag IDs @@ -145,8 +171,8 @@ public: */ inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const { - const CState *c = _caps.get(id); - return ((c) ? (((c->lastReceived != 0)&&(c->cap.expiration() < nconf.timestamp)) ? &(c->cap) : (const Capability *)0) : (const Capability *)0); + std::map::const_iterator c(_caps.find(id)); + return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(c->second.cap.expiration() < nconf.timestamp)) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); } /** @@ -179,18 +205,18 @@ public: { uint64_t lastAct = _lastPushedCom; - uint32_t *i = (uint32_t *)0; - CState *cs = (CState *)0; - Hashtable::Iterator csi(_caps); - while (csi.next(i,cs)) { - const uint64_t la = std::max(cs->lastPushed,cs->lastReceived); - if ((now - la) > ZT_MEMBERSHIP_STATE_EXPIRATION_TIME) - _caps.erase(*i); - else if (la > lastAct) - lastAct = la; + for(std::map::iterator i(_caps.begin());i!=_caps.end();) { + const uint64_t la = std::max(i->second.lastPushed,i->second.lastReceived); + if ((now - la) > ZT_MEMBERSHIP_STATE_EXPIRATION_TIME) { + _caps.erase(i++); + } else { + ++i; + if (la > lastAct) + lastAct = la; + } } - i = (uint32_t *)0; + uint32_t *i = (uint32_t *)0; TState *ts = (TState *)0; Hashtable::Iterator tsi(_tags); while (tsi.next(i,ts)) { @@ -211,8 +237,8 @@ private: // COM from this peer CertificateOfMembership _com; - // Capability-related state - Hashtable _caps; + // Capability-related state (we need an ordered container here, hence std::map) + std::map _caps; // Tag-related state Hashtable _tags; diff --git a/node/Network.cpp b/node/Network.cpp index 1c894306..a8165d3e 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -66,7 +66,8 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } -static bool _doZtFilter( +// 0 == no match, -1 == match/drop, 1 == match/accept +static int _doZtFilter( const RuntimeEnvironment *RR, const uint64_t nwid, const bool inbound, @@ -99,18 +100,21 @@ static bool _doZtFilter( switch(rt) { // Actions ------------------------------------------------------------- + // An action is performed if thisSetMatches is true, and if not + // (or if the action is non-terminating) we start a new set of rules. + case ZT_NETWORK_RULE_ACTION_DROP: if (thisSetMatches) { - return false; + return -1; // match, drop packet } else { - thisSetMatches = 1; // continue parsing next set of rules + thisSetMatches = 1; // no match, evaluate next set } break; case ZT_NETWORK_RULE_ACTION_ACCEPT: if (thisSetMatches) { - return true; + return 1; // match, accept packet } else { - thisSetMatches = 1; // continue parsing next set of rules + thisSetMatches = 1; // no match, evaluate next set } break; case ZT_NETWORK_RULE_ACTION_TEE: @@ -126,14 +130,16 @@ static bool _doZtFilter( RR->sw->send(outp,true,nwid); if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { - return false; + return -1; // match, drop packet (we redirected it) } else { - thisSetMatches = 1; // TEE does not terminate parsing + thisSetMatches = 1; // TEE does not terminate evaluation } } break; // Rules --------------------------------------------------------------- + // thisSetMatches is the binary AND of the result of all rules in a set + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); break; @@ -320,13 +326,13 @@ static bool _doZtFilter( } break; } - // thisSetMatches remains true if the current rule matched... or does NOT match if not bit (0x80) is 1 + // thisSetMatches remains true if the current rule matched (or did NOT match if NOT bit is set) thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); //TRACE("[%u] %u result==%u set==%u",rn,(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); } - return false; + return 0; } const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); @@ -414,58 +420,22 @@ bool Network::filterOutgoingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - if (_doZtFilter( - RR, - _id, - false, - ztSource, - ztDest, - macSource, - macDest, - frameData, - frameLen, - etherType, - vlanId, - _config.rules, - _config.ruleCount, - _config.tags, - _config.tagCount, - remoteTagIds, - remoteTagValues, - remoteTagCount, - relevantLocalTags, - relevantLocalTagCount - )) { - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); - return true; + switch(_doZtFilter(RR,_id,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + case -1: + return false; + case 1: + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); + return true; } for(unsigned int c=0;c<_config.capabilityCount;++c) { relevantLocalTagCount = 0; - if (_doZtFilter( - RR, - _id, - false, - ztSource, - ztDest, - macSource, - macDest, - frameData, - frameLen, - etherType, - vlanId, - _config.capabilities[c].rules(), - _config.capabilities[c].ruleCount(), - _config.tags, - _config.tagCount, - remoteTagIds, - remoteTagValues, - remoteTagCount, - relevantLocalTags, - relevantLocalTagCount - )) { - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); - return true; + switch (_doZtFilter(RR,_id,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + case -1: + return false; + case 1: + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); + return true; } } @@ -492,32 +462,24 @@ bool Network::filterIncomingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - if (_doZtFilter( - RR, - _id, - true, - sourcePeer->address(), - ztDest, - macSource, - macDest, - frameData, - frameLen, - etherType, - vlanId, - _config.rules, - _config.ruleCount, - _config.tags, - _config.tagCount, - remoteTagIds, - remoteTagValues, - remoteTagCount, - relevantLocalTags, - relevantLocalTagCount - )) { - return true; + switch (_doZtFilter(RR,_id,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + case -1: + return false; + case 1: + return true; } - + Membership::CapabilityIterator mci(m); + const Capability *c; + while ((c = mci.next())) { + relevantLocalTagCount = 0; + switch(_doZtFilter(RR,_id,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + case -1: + return false; + case 1: + return true; + } + } return false; } @@ -817,10 +779,9 @@ bool Network::_isAllowed(const SharedPtr &peer) const if (_config.isPublic()) { return true; } else { - LockingPtr m(peer->membership(_id,false)); - if (m) { + const Membership *m = _memberships.get(peer->address()); + if (m) return _config.com.agreesWith(m->com()); - } } } } catch ( ... ) { @@ -866,14 +827,15 @@ void Network::_announceMulticastGroups() _announceMulticastGroupsTo(*i,allMulticastGroups); } -void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std::vector &allMulticastGroups) const +void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std::vector &allMulticastGroups) { // Assumes _lock is locked // Anyone we announce multicast groups to will need our COM to authenticate GATHER requests. { - LockingPtr m(peer->membership(_id,false)); - if (m) m->sendCredentialsIfNeeded(RR,RR->node->now(),*peer,_config); + Membership *m = _memberships.get(peer->address()); + if (m) + m->sendCredentialsIfNeeded(RR,RR->node->now(),peer->address(),_config.com,(const Capability *)0,(const Tag **)0,0); } Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); diff --git a/node/Network.hpp b/node/Network.hpp index a8eb3156..06fd7735 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -362,7 +362,7 @@ private: void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _isAllowed(const SharedPtr &peer) const; void _announceMulticastGroups(); - void _announceMulticastGroupsTo(const SharedPtr &peer,const std::vector &allMulticastGroups) const; + void _announceMulticastGroupsTo(const SharedPtr &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; const RuntimeEnvironment *RR; -- cgit v1.2.3 From 00fd9c3a15f9ac0981cf79c98515df888b3bd109 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 Aug 2016 17:33:26 -0700 Subject: It builds... almost ready to test some rules engine stuff. --- node/Capability.hpp | 35 ++++++++++++++++++------------- node/IncomingPacket.cpp | 55 +++++++++++++++++++++++++------------------------ node/Membership.cpp | 22 +++++++++++--------- node/Membership.hpp | 6 +++--- node/Multicaster.cpp | 15 +------------- node/Multicaster.hpp | 2 -- node/Network.cpp | 2 +- node/Network.hpp | 36 ++++++++++++++++++++++++++++++++ node/Packet.cpp | 5 ++--- node/Packet.hpp | 4 ++-- node/Peer.cpp | 4 +--- node/Switch.cpp | 53 +++++++---------------------------------------- node/Tag.hpp | 8 ++++++- node/Topology.cpp | 28 ------------------------- 14 files changed, 121 insertions(+), 154 deletions(-) (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index 53457d4d..42d4ce63 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -71,16 +71,18 @@ public: /** * @param id Capability ID * @param nwid Network ID + * @param ts Timestamp (at controller) * @param expiration Expiration relative to network config timestamp * @param name Capability short name (max strlen == ZT_MAX_CAPABILITY_NAME_LENGTH, overflow ignored) * @param mccl Maximum custody chain length (1 to create non-transferrable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,uint64_t expiration,const char *name,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,const char *name,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { memset(this,0,sizeof(Capability)); _nwid = nwid; + _ts = ts; _expiration = expiration; _id = id; _maxCustodyChainLength = (mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1; @@ -115,20 +117,22 @@ public: inline uint64_t expiration() const { return _expiration; } /** - * Check to see if a given address is a 'to' address in the custody chain - * - * This does not actually do certificate checking. That must be done with verify(). - * - * @param a Address to check - * @return True if address is present + * @return Timestamp + */ + inline uint64_t timestamp() const { return _ts; } + + /** + * @return Last 'to' address in chain of custody */ - inline bool wasIssuedTo(const Address &a) const + inline Address issuedTo() const { + Address i2; for(unsigned int i=0;i(p); p += 4; _nwid = b.template at(p); p += 8; + _ts = b.template at(p); p += 8; _expiration = b.template at(p); p += 8; + _id = b.template at(p); p += 4; deserializeRules(b,p,_rules,_ruleCount,ZT_MAX_CAPABILITY_RULES); _maxCustodyChainLength = (unsigned int)b[p++]; if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) throw std::runtime_error("invalid max custody chain length"); - for(unsigned int i;;++i) { + for(unsigned int i=0;;++i) { const Address to(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (!to) break; @@ -409,6 +415,7 @@ public: private: uint64_t _nwid; + uint64_t _ts; uint64_t _expiration; uint32_t _id; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c2df7ee2..29d0964c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -437,8 +437,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p if ((flags & 0x01) != 0) { // deprecated but still used by older peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); - LockingPtr m(peer->membership(com.networkId(),true)); - if (m) m->addCredential(RR,RR->node->now(),com); + if (com) { + SharedPtr network(RR->node->network(com.networkId())); + if (network) + network->addCredential(com); + } } if ((flags & 0x02) != 0) { @@ -567,8 +570,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

m(peer->membership(com.networkId(),true)); - if (m) m->addCredential(RR,RR->node->now(),com); + if (com) + network->addCredential(com); } if (!network->isAllowed(peer)) { @@ -661,7 +664,6 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - const uint64_t now = RR->node->now(); CertificateOfMembership com; Capability cap; Tag tag; @@ -669,9 +671,13 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S unsigned int p = ZT_PACKET_IDX_PAYLOAD; while ((p < size())&&((*this)[p])) { p += com.deserialize(*this,p); - LockingPtr m(peer->membership(com.networkId(),true)); - if (!m) return true; // sanity check - if (m->addCredential(RR,now,com) == 1) return false; // wait for WHOIS + if (com) { + SharedPtr network(RR->node->network(com.networkId())); + if (network) { + if (network->addCredential(com) == 1) + return false; // wait for WHOIS + } + } } ++p; // skip trailing 0 after COMs if present @@ -679,17 +685,21 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S const unsigned int numCapabilities = at(p); p += 2; for(unsigned int i=0;i m(peer->membership(cap.networkId(),true)); - if (!m) return true; // sanity check - if (m->addCredential(RR,now,cap) == 1) return false; // wait for WHOIS + SharedPtr network(RR->node->network(cap.networkId())); + if (network) { + if (network->addCredential(cap) == 1) + return false; // wait for WHOIS + } } const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i m(peer->membership(tag.networkId(),true)); - if (!m) return true; // sanity check - if (m->addCredential(RR,now,tag) == 1) return false; // wait for WHOIS + SharedPtr network(RR->node->network(tag.networkId())); + if (network) { + if (network->addCredential(tag) == 1) + return false; // wait for WHOIS + } } } @@ -830,8 +840,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((flags & 0x01) != 0) { // deprecated but still used by older peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); - LockingPtr m(peer->membership(com.networkId(),true)); - if (m) m->addCredential(RR,RR->node->now(),com); + if (com) + network->addCredential(com); } // Check membership after we've read any included COM, since @@ -1037,17 +1047,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt NetworkConfig originatorCredentialNetworkConfig; if (originatorCredentialNetworkId) { if (Network::controllerFor(originatorCredentialNetworkId) == originatorAddress) { - SharedPtr nw(RR->node->network(originatorCredentialNetworkId)); - if ((nw)&&(nw->hasConfig())) { - originatorCredentialNetworkConfig = nw->config(); - if ( ( (originatorCredentialNetworkConfig.isPublic()) || (peer->address() == originatorAddress) || ((originatorCredentialNetworkConfig.com)&&(previousHopCom)&&(originatorCredentialNetworkConfig.com.agreesWith(previousHopCom))) ) ) { - TRACE("CIRCUIT_TEST %.16llx received from hop %s(%s) and originator %s with valid network ID credential %.16llx (verified from originator and next hop)",testId,source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - } else { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and previous hop %s did not supply a valid COM",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId,peer->address().toString().c_str()); - return true; - } - } else { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we are not a member",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); + if (!RR->node->network(originatorCredentialNetworkId)) { + TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we are not a member of that network",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); return true; } } else { diff --git a/node/Membership.cpp b/node/Membership.cpp index 79b1e1bc..e12bce3c 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -79,19 +79,19 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint return false; } -int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com) +int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) { if (com.issuedTo() != RR->identity.address()) return -1; if (_com == com) return 0; const int vr = com.verify(RR); - if (vr == 0) + if ((vr == 0)&&(com.revision() > _com.revision())) _com = com; return vr; } -int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Tag &tag) +int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) { if (tag.issuedTo() != RR->identity.address()) return -1; @@ -102,15 +102,17 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,co if (vr == 0) { if (!t) t = &(_tags[tag.id()]); - t->lastReceived = now; - t->tag = tag; + if (t->tag.timestamp() <= tag.timestamp()) { + t->lastReceived = RR->node->now(); + t->tag = tag; + } } return vr; } -int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Capability &cap) +int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap) { - if (!cap.wasIssuedTo(RR->identity.address())) + if (cap.issuedTo() != RR->identity.address()) return -1; std::map::iterator c(_caps.find(cap.id())); if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) @@ -119,10 +121,10 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const uint64_t now,co if (vr == 0) { if (c == _caps.end()) { CState &c2 = _caps[cap.id()]; - c2.lastReceived = now; + c2.lastReceived = RR->node->now(); c2.cap = cap; - } else { - c->second.lastReceived = now; + } else if (c->second.cap.timestamp() <= cap.timestamp()) { + c->second.lastReceived = RR->node->now(); c->second.cap = cap; } } diff --git a/node/Membership.hpp b/node/Membership.hpp index 664cd2ad..e9f9d488 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -180,21 +180,21 @@ public: * * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const CertificateOfMembership &com); + int addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good * * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Tag &tag); + int addCredential(const RuntimeEnvironment *RR,const Tag &tag); /** * Validate and add a credential if signature is okay and it's otherwise good * * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const uint64_t now,const Capability &cap); + int addCredential(const RuntimeEnvironment *RR,const Capability &cap); /** * Clean up old or stale entries diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index e1d4567a..9e583e34 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -152,7 +152,6 @@ std::vector

Multicaster::getMembers(uint64_t nwid,const MulticastGroup } void Multicaster::send( - const CertificateOfMembership *com, unsigned int limit, uint64_t now, uint64_t nwid, @@ -194,7 +193,6 @@ void Multicaster::send( RR, now, nwid, - com, limit, 1, // we'll still gather a little from peers to keep multicast list fresh src, @@ -236,22 +234,12 @@ void Multicaster::send( if (!p) continue; //TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str()); - - const CertificateOfMembership *com = (CertificateOfMembership *)0; - { - SharedPtr nw(RR->node->network(nwid)); - if ((nw)&&(nw->hasConfig())&&(nw->config().com)&&(nw->config().isPrivate())&&(p->needsOurNetworkMembershipCertificate(nwid,now,true))) - com = &(nw->config().com); - } - Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER); outp.append(nwid); - outp.append((uint8_t)(com ? 0x01 : 0x00)); + outp.append((uint8_t)0x00); mg.mac().appendTo(outp); outp.append((uint32_t)mg.adi()); outp.append((uint32_t)gatherLimit); - if (com) - com->serialize(outp); RR->sw->send(outp,true,0); } gatherLimit = 0; @@ -264,7 +252,6 @@ void Multicaster::send( RR, now, nwid, - com, limit, gatherLimit, src, diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index c43c8d93..51dabc69 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -150,7 +150,6 @@ public: /** * Send a multicast * - * @param com Certificate of membership to include or NULL for none * @param limit Multicast limit * @param now Current time * @param nwid Network ID @@ -162,7 +161,6 @@ public: * @param len Length of packet data */ void send( - const CertificateOfMembership *com, unsigned int limit, uint64_t now, uint64_t nwid, diff --git a/node/Network.cpp b/node/Network.cpp index a8165d3e..fd2fac2b 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -348,7 +348,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _netconfFailure(NETCONF_FAILURE_NONE), _portError(0) { - char confn[128],mcdbn[128]; + char confn[128]; Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id); if (_id == ZT_TEST_NETWORK_ID) { diff --git a/node/Network.hpp b/node/Network.hpp index 06fd7735..16f07163 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -336,6 +336,42 @@ public: */ void learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now); + /** + * @param com Certificate of membership + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + inline int addCredential(const CertificateOfMembership &com) + { + if (com.networkId() != _id) + return -1; + Mutex::Lock _l(_lock); + return _memberships[com.issuedTo()].addCredential(RR,com); + } + + /** + * @param cap Capability + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + inline int addCredential(const Capability &cap) + { + if (cap.networkId() != _id) + return -1; + Mutex::Lock _l(_lock); + return _memberships[cap.issuedTo()].addCredential(RR,cap); + } + + /** + * @param cap Tag + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + inline int addCredential(const Tag &tag) + { + if (tag.networkId() != _id) + return -1; + Mutex::Lock _l(_lock); + return _memberships[tag.issuedTo()].addCredential(RR,tag); + } + /** * Destroy this network * diff --git a/node/Packet.cpp b/node/Packet.cpp index 4aebf6a9..eda60757 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -22,7 +22,7 @@ namespace ZeroTier { const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; -//#ifdef ZT_TRACE +#ifdef ZT_TRACE const char *Packet::verbString(Verb v) throw() @@ -60,14 +60,13 @@ const char *Packet::errorString(ErrorCode e) case ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND"; case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION"; case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION"; - case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE"; case ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED"; case ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST"; } return "(unknown)"; } -//#endif // ZT_TRACE +#endif // ZT_TRACE void Packet::armor(const void *key,bool encryptPayload) { diff --git a/node/Packet.hpp b/node/Packet.hpp index 6789580e..dce9f208 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1060,12 +1060,12 @@ public: ERROR_UNWANTED_MULTICAST = 0x08 }; -//#ifdef ZT_TRACE +#ifdef ZT_TRACE static const char *verbString(Verb v) throw(); static const char *errorString(ErrorCode e) throw(); -//#endif +#endif template Packet(const Buffer &b) : diff --git a/node/Peer.cpp b/node/Peer.cpp index ba47a0be..89dce570 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -53,9 +53,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _id(peerIdentity), _numPaths(0), _latency(0), - _directPathPushCutoffCount(0), - _networkComs(4), - _lastPushedComs(4) + _directPathPushCutoffCount(0) { if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw std::runtime_error("new peer identity key agreement failed"); diff --git a/node/Switch.cpp b/node/Switch.cpp index 33b08429..167c7928 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -35,7 +35,6 @@ #include "Peer.hpp" #include "SelfAwareness.hpp" #include "Packet.hpp" -#include "Filter.hpp" #include "Cluster.hpp" namespace ZeroTier { @@ -438,26 +437,12 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),len); - if (!Filter::run( - RR, - network->id(), - RR->identity.address(), - Address(), // 0 destination ZT address for multicasts since this is unknown at time of send - from, - to, - (const uint8_t *)data, - len, - etherType, - vlanId, - network->config().rules, - network->config().ruleCount)) - { - TRACE("%.16llx: %s -> %s %s packet not sent: Filter::run() == false (multicast)",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + if (!network->filterOutgoingPacket(RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } RR->mc->send( - ((!network->config().isPublic())&&(network->config().com)) ? &(network->config().com) : (const CertificateOfMembership *)0, network->config().multicastLimit, RR->node->now(), network->id(), @@ -477,34 +462,15 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this SharedPtr toPeer(RR->topology->getPeer(toZT)); - if (!Filter::run( - RR, - network->id(), - RR->identity.address(), - toZT, - from, - to, - (const uint8_t *)data, - len, - etherType, - vlanId, - network->config().rules, - network->config().ruleCount)) - { - TRACE("%.16llx: %s -> %s %s packet not sent: Filter::run() == false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + if (!network->filterOutgoingPacket(RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { + TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } - const bool includeCom = ( (network->config().isPrivate()) && (network->config().com) && ((!toPeer)||(toPeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) ); - if ((fromBridged)||(includeCom)) { + if (fromBridged) { Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(network->id()); - if (includeCom) { - outp.append((unsigned char)0x01); // 0x01 -- COM included - network->config().com.serialize(outp); - } else { - outp.append((unsigned char)0x00); - } + outp.append((unsigned char)0x00); to.appendTo(outp); from.appendTo(outp); outp.append((uint16_t)etherType); @@ -564,12 +530,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c SharedPtr bridgePeer(RR->topology->getPeer(bridges[b])); Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(network->id()); - if ( (network->config().isPrivate()) && (network->config().com) && ((!bridgePeer)||(bridgePeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) ) { - outp.append((unsigned char)0x01); // 0x01 -- COM included - network->config().com.serialize(outp); - } else { - outp.append((unsigned char)0); - } + outp.append((uint8_t)0x00); to.appendTo(outp); from.appendTo(outp); outp.append((uint16_t)etherType); diff --git a/node/Tag.hpp b/node/Tag.hpp index a9f6f57e..b4bc63c4 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -61,13 +61,15 @@ public: /** * @param nwid Network ID + * @param ts Timestamp * @param expiration Tag expiration relative to network config timestamp * @param issuedTo Address to which this tag was issued * @param id Tag ID * @param value Tag value */ - Tag(const uint64_t nwid,const uint64_t expiration,const Address &issuedTo,const uint32_t id,const uint32_t value) : + Tag(const uint64_t nwid,const uint64_t ts,const uint64_t expiration,const Address &issuedTo,const uint32_t id,const uint32_t value) : _nwid(nwid), + _ts(ts), _expiration(expiration), _id(id), _value(value), @@ -78,6 +80,7 @@ public: inline uint64_t networkId() const { return _nwid; } inline uint64_t expiration() const { return _expiration; } + inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } inline const Address &issuedTo() const { return _issuedTo; } @@ -115,6 +118,7 @@ public: if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); b.append(_nwid); + b.append(_ts); b.append(_expiration); b.append(_id); b.append(_value); @@ -136,6 +140,7 @@ public: unsigned int p = startAt; _nwid = b.template at(p); p += 8; + _ts = b.template at(p); p += 8; _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; _value = b.template at(p); p += 4; @@ -163,6 +168,7 @@ public: private: uint64_t _nwid; + uint64_t _ts; uint64_t _expiration; uint32_t _id; uint32_t _value; diff --git a/node/Topology.cpp b/node/Topology.cpp index ef1c1698..6e0fe90c 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -74,34 +74,6 @@ Topology::Topology(const RuntimeEnvironment *renv) : Topology::~Topology() { - Buffer *pbuf = 0; - try { - pbuf = new Buffer(); - std::string all; - - Address *a = (Address *)0; - SharedPtr *p = (SharedPtr *)0; - Hashtable< Address,SharedPtr >::Iterator i(_peers); - while (i.next(a,p)) { - if (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) { - pbuf->clear(); - try { - (*p)->serialize(*pbuf); - try { - all.append((const char *)pbuf->data(),pbuf->size()); - } catch ( ... ) { - return; // out of memory? just skip - } - } catch ( ... ) {} // peer too big? shouldn't happen, but it so skip - } - } - - RR->node->dataStorePut("peers.save",all,true); - - delete pbuf; - } catch ( ... ) { - delete pbuf; - } } SharedPtr Topology::addPeer(const SharedPtr &peer) -- cgit v1.2.3 From 51cf49a24fa5e953de0192006c624675090483f5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 Aug 2016 17:40:22 -0700 Subject: cleanup --- attic/BinarySemaphore.hpp | 97 +++++++++++++++++++++++++++++++++++++++++++++++ node/BinarySemaphore.hpp | 97 ----------------------------------------------- 2 files changed, 97 insertions(+), 97 deletions(-) create mode 100644 attic/BinarySemaphore.hpp delete mode 100644 node/BinarySemaphore.hpp (limited to 'node') diff --git a/attic/BinarySemaphore.hpp b/attic/BinarySemaphore.hpp new file mode 100644 index 00000000..315d2b00 --- /dev/null +++ b/attic/BinarySemaphore.hpp @@ -0,0 +1,97 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_BINARYSEMAPHORE_HPP +#define ZT_BINARYSEMAPHORE_HPP + +#include +#include +#include + +#include "Constants.hpp" +#include "NonCopyable.hpp" + +#ifdef __WINDOWS__ + +#include + +namespace ZeroTier { + +class BinarySemaphore : NonCopyable +{ +public: + BinarySemaphore() throw() { _sem = CreateSemaphore(NULL,0,1,NULL); } + ~BinarySemaphore() { CloseHandle(_sem); } + inline void wait() { WaitForSingleObject(_sem,INFINITE); } + inline void post() { ReleaseSemaphore(_sem,1,NULL); } +private: + HANDLE _sem; +}; + +} // namespace ZeroTier + +#else // !__WINDOWS__ + +#include + +namespace ZeroTier { + +class BinarySemaphore : NonCopyable +{ +public: + BinarySemaphore() + { + pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0); + pthread_cond_init(&_cond,(const pthread_condattr_t *)0); + _f = false; + } + + ~BinarySemaphore() + { + pthread_cond_destroy(&_cond); + pthread_mutex_destroy(&_mh); + } + + inline void wait() + { + pthread_mutex_lock(const_cast (&_mh)); + while (!_f) + pthread_cond_wait(const_cast (&_cond),const_cast (&_mh)); + _f = false; + pthread_mutex_unlock(const_cast (&_mh)); + } + + inline void post() + { + pthread_mutex_lock(const_cast (&_mh)); + _f = true; + pthread_mutex_unlock(const_cast (&_mh)); + pthread_cond_signal(const_cast (&_cond)); + } + +private: + pthread_cond_t _cond; + pthread_mutex_t _mh; + volatile bool _f; +}; + +} // namespace ZeroTier + +#endif // !__WINDOWS__ + +#endif diff --git a/node/BinarySemaphore.hpp b/node/BinarySemaphore.hpp deleted file mode 100644 index 315d2b00..00000000 --- a/node/BinarySemaphore.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef ZT_BINARYSEMAPHORE_HPP -#define ZT_BINARYSEMAPHORE_HPP - -#include -#include -#include - -#include "Constants.hpp" -#include "NonCopyable.hpp" - -#ifdef __WINDOWS__ - -#include - -namespace ZeroTier { - -class BinarySemaphore : NonCopyable -{ -public: - BinarySemaphore() throw() { _sem = CreateSemaphore(NULL,0,1,NULL); } - ~BinarySemaphore() { CloseHandle(_sem); } - inline void wait() { WaitForSingleObject(_sem,INFINITE); } - inline void post() { ReleaseSemaphore(_sem,1,NULL); } -private: - HANDLE _sem; -}; - -} // namespace ZeroTier - -#else // !__WINDOWS__ - -#include - -namespace ZeroTier { - -class BinarySemaphore : NonCopyable -{ -public: - BinarySemaphore() - { - pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0); - pthread_cond_init(&_cond,(const pthread_condattr_t *)0); - _f = false; - } - - ~BinarySemaphore() - { - pthread_cond_destroy(&_cond); - pthread_mutex_destroy(&_mh); - } - - inline void wait() - { - pthread_mutex_lock(const_cast (&_mh)); - while (!_f) - pthread_cond_wait(const_cast (&_cond),const_cast (&_mh)); - _f = false; - pthread_mutex_unlock(const_cast (&_mh)); - } - - inline void post() - { - pthread_mutex_lock(const_cast (&_mh)); - _f = true; - pthread_mutex_unlock(const_cast (&_mh)); - pthread_cond_signal(const_cast (&_cond)); - } - -private: - pthread_cond_t _cond; - pthread_mutex_t _mh; - volatile bool _f; -}; - -} // namespace ZeroTier - -#endif // !__WINDOWS__ - -#endif -- cgit v1.2.3 From 2ba93436077b4f4901db81687df2e03d7ce6c8c5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 08:32:42 -0700 Subject: Encode and decode of tags and capabilities in NetworkConfig. --- node/Capability.hpp | 74 ++++---- node/IncomingPacket.cpp | 2 +- node/Network.cpp | 15 +- node/NetworkConfig.cpp | 436 +++++++++++++++++++-------------------------- node/NetworkConfig.hpp | 36 +++- node/NetworkController.hpp | 2 +- node/Tag.hpp | 7 + 7 files changed, 267 insertions(+), 305 deletions(-) (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index 42d4ce63..fd6ae091 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -264,40 +264,6 @@ public: } } - template - inline void serialize(Buffer &b,const bool forSign = false) const - { - if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - - b.append(_nwid); - b.append(_ts); - b.append(_expiration); - b.append(_id); - serializeRules(b,_rules,_ruleCount); - b.append((uint8_t)_maxCustodyChainLength); - - if (!forSign) { - for(unsigned int i=0;;++i) { - if ((i < _maxCustodyChainLength)&&(i < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)&&(_custody[i].to)) { - _custody[i].to.appendTo(b); - _custody[i].from.appendTo(b); - b.append((uint8_t)1); // 1 == Ed25519 signature - b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); // length of signature - b.append(_custody[i].signature.data,ZT_C25519_SIGNATURE_LEN); - } else { - b.append((unsigned char)0,ZT_ADDRESS_LENGTH); // zero 'to' terminates chain - break; - } - } - } - - // This is the size of any additional fields. If it is nonzero, - // the last 2 bytes of the next field will be another size field. - b.append((uint16_t)0); - - if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - } - template static inline void deserializeRules(const Buffer &b,unsigned int &p,ZT_VirtualNetworkRule *rules,unsigned int &ruleCount,const unsigned int maxRuleCount) { @@ -373,6 +339,41 @@ public: } } + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + // These are the same between Tag and Capability + b.append(_nwid); + b.append(_ts); + b.append(_expiration); + b.append(_id); + + serializeRules(b,_rules,_ruleCount); + b.append((uint8_t)_maxCustodyChainLength); + + if (!forSign) { + for(unsigned int i=0;;++i) { + if ((i < _maxCustodyChainLength)&&(i < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)&&(_custody[i].to)) { + _custody[i].to.appendTo(b); + _custody[i].from.appendTo(b); + b.append((uint8_t)1); // 1 == Ed25519 signature + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); // length of signature + b.append(_custody[i].signature.data,ZT_C25519_SIGNATURE_LEN); + } else { + b.append((unsigned char)0,ZT_ADDRESS_LENGTH); // zero 'to' terminates chain + break; + } + } + } + + // This is the size of any additional fields, currently 0. + b.append((uint16_t)0); + + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + } + template inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) { @@ -380,15 +381,18 @@ public: unsigned int p = startAt; + // These are the same between Tag and Capability _nwid = b.template at(p); p += 8; _ts = b.template at(p); p += 8; _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; + deserializeRules(b,p,_rules,_ruleCount,ZT_MAX_CAPABILITY_RULES); - _maxCustodyChainLength = (unsigned int)b[p++]; + _maxCustodyChainLength = (unsigned int)b[p++]; if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) throw std::runtime_error("invalid max custody chain length"); + for(unsigned int i=0;;++i) { const Address to(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (!to) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 29d0964c..fae689d1 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -717,7 +717,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); - const Dictionary metaData(metaDataBytes,metaDataLength); + const Dictionary metaData(metaDataBytes,metaDataLength); //const uint64_t haveRevision = ((ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength + 8) <= size()) ? at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength) : 0ULL; diff --git a/node/Network.cpp b/node/Network.cpp index fd2fac2b..0fbdf5ba 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -358,18 +358,21 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : RR->node->dataStorePut(confn,"\n",1,false); } else { bool gotConf = false; + Dictionary *dconf = new Dictionary(); + NetworkConfig *nconf = new NetworkConfig(); try { std::string conf(RR->node->dataStoreGet(confn)); if (conf.length()) { - Dictionary dconf(conf.c_str()); - NetworkConfig nconf; - if (nconf.fromDictionary(dconf)) { - this->setConfiguration(nconf,false); + dconf->load(conf.c_str()); + if (nconf->fromDictionary(*dconf)) { + this->setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } } } catch ( ... ) {} // ignore invalids, we'll re-request + delete nconf; + delete dconf; if (!gotConf) { // Save a one-byte CR to persist membership while we request a real netconf @@ -591,14 +594,16 @@ void Network::requestConfiguration() if (_id == ZT_TEST_NETWORK_ID) // pseudo-network-ID, uses locally generated static config return; - Dictionary rmd; + Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MINOR); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,(uint64_t)ZEROTIER_ONE_VERSION_REVISION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES,(uint64_t)ZT_MAX_NETWORK_RULES); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_CAPABILITIES,(uint64_t)ZT_MAX_NETWORK_CAPABILITIES); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES,(uint64_t)ZT_MAX_CAPABILITY_RULES); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); if (controller() == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index a8ab4dac..07e9bd4f 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -18,6 +18,8 @@ #include +#include + #include "NetworkConfig.hpp" #include "Utils.hpp" @@ -25,204 +27,152 @@ namespace ZeroTier { bool NetworkConfig::toDictionary(Dictionary &d,bool includeLegacy) const { - Buffer tmp; + Buffer *tmp = new Buffer(); - d.clear(); + try { + d.clear(); - // Try to put the more human-readable fields first + // Try to put the more human-readable fields first - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name)) return false; #ifdef ZT_SUPPORT_OLD_STYLE_NETCONF - if (includeLegacy) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ALLOW_PASSIVE_BRIDGING_OLD,this->allowPassiveBridging())) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST_OLD,this->enableBroadcast())) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_PRIVATE_OLD,this->isPrivate())) return false; - - std::string v4s; - for(unsigned int i=0;istaticIps[i].ss_family == AF_INET) { - if (v4s.length() > 0) - v4s.push_back(','); - v4s.append(this->staticIps[i].toString()); + if (includeLegacy) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ALLOW_PASSIVE_BRIDGING_OLD,this->allowPassiveBridging())) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST_OLD,this->enableBroadcast())) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_PRIVATE_OLD,this->isPrivate())) return false; + + std::string v4s; + for(unsigned int i=0;istaticIps[i].ss_family == AF_INET) { + if (v4s.length() > 0) + v4s.push_back(','); + v4s.append(this->staticIps[i].toString()); + } } - } - if (v4s.length() > 0) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_IPV4_STATIC_OLD,v4s.c_str())) return false; - } - std::string v6s; - for(unsigned int i=0;istaticIps[i].ss_family == AF_INET6) { - if (v6s.length() > 0) - v6s.push_back(','); - v6s.append(this->staticIps[i].toString()); + if (v4s.length() > 0) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_IPV4_STATIC_OLD,v4s.c_str())) return false; + } + std::string v6s; + for(unsigned int i=0;istaticIps[i].ss_family == AF_INET6) { + if (v6s.length() > 0) + v6s.push_back(','); + v6s.append(this->staticIps[i].toString()); + } + } + if (v6s.length() > 0) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_IPV6_STATIC_OLD,v6s.c_str())) return false; } - } - if (v6s.length() > 0) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_IPV6_STATIC_OLD,v6s.c_str())) return false; - } - std::string ets; - unsigned int et = 0; - ZT_VirtualNetworkRuleType lastrt = ZT_NETWORK_RULE_ACTION_ACCEPT; - for(unsigned int i=0;i 0) - ets.push_back(','); - char tmp[16]; - Utils::snprintf(tmp,sizeof(tmp),"%x",et); - ets.append(tmp); + std::string ets; + unsigned int et = 0; + ZT_VirtualNetworkRuleType lastrt = ZT_NETWORK_RULE_ACTION_ACCEPT; + for(unsigned int i=0;i 0) + ets.push_back(','); + char tmp2[16]; + Utils::snprintf(tmp2,sizeof(tmp2),"%x",et); + ets.append(tmp2); + } + et = 0; } - et = 0; + lastrt = rt; + } + if (ets.length() > 0) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES_OLD,ets.c_str())) return false; } - lastrt = rt; - } - if (ets.length() > 0) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES_OLD,ets.c_str())) return false; - } - if (this->com) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATE_OF_MEMBERSHIP_OLD,this->com.toString().c_str())) return false; - } + if (this->com) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATE_OF_MEMBERSHIP_OLD,this->com.toString().c_str())) return false; + } - std::string ab; - for(unsigned int i=0;ispecialistCount;++i) { - if ((this->specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0) { - if (ab.length() > 0) - ab.push_back(','); - ab.append(Address(this->specialists[i]).toString().c_str()); + std::string ab; + for(unsigned int i=0;ispecialistCount;++i) { + if ((this->specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0) { + if (ab.length() > 0) + ab.push_back(','); + ab.append(Address(this->specialists[i]).toString().c_str()); + } + } + if (ab.length() > 0) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ACTIVE_BRIDGES_OLD,ab.c_str())) return false; } } - if (ab.length() > 0) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ACTIVE_BRIDGES_OLD,ab.c_str())) return false; - } - } #endif // ZT_SUPPORT_OLD_STYLE_NETCONF - // Then add binary blobs + // Then add binary blobs - if (this->com) { - tmp.clear(); - this->com.serialize(tmp); - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_COM,tmp)) return false; - } + if (this->com) { + tmp->clear(); + this->com.serialize(*tmp); + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_COM,*tmp)) return false; + } - tmp.clear(); - for(unsigned int i=0;ispecialistCount;++i) { - tmp.append((uint64_t)this->specialists[i]); - } - if (tmp.size()) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_SPECIALISTS,tmp)) return false; - } + tmp->clear(); + for(unsigned int i=0;icapabilityCount;++i) + this->capabilities[i].serialize(*tmp); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CAPABILITIES,*tmp)) return false; + } - tmp.clear(); - for(unsigned int i=0;irouteCount;++i) { - reinterpret_cast(&(this->routes[i].target))->serialize(tmp); - reinterpret_cast(&(this->routes[i].via))->serialize(tmp); - tmp.append((uint16_t)this->routes[i].flags); - tmp.append((uint16_t)this->routes[i].metric); - } - if (tmp.size()) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ROUTES,tmp)) return false; - } + tmp->clear(); + for(unsigned int i=0;itagCount;++i) + this->tags[i].serialize(*tmp); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TAGS,*tmp)) return false; + } - tmp.clear(); - for(unsigned int i=0;istaticIpCount;++i) { - this->staticIps[i].serialize(tmp); - } - if (tmp.size()) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_STATIC_IPS,tmp)) return false; - } + tmp->clear(); + for(unsigned int i=0;ispecialistCount;++i) + tmp->append((uint64_t)this->specialists[i]); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_SPECIALISTS,*tmp)) return false; + } - tmp.clear(); - for(unsigned int i=0;iruleCount;++i) { - tmp.append((uint8_t)rules[i].t); - switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x7f)) { - //case ZT_NETWORK_RULE_ACTION_DROP: - //case ZT_NETWORK_RULE_ACTION_ACCEPT: - default: - tmp.append((uint8_t)0); - break; - case ZT_NETWORK_RULE_ACTION_TEE: - case ZT_NETWORK_RULE_ACTION_REDIRECT: - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - tmp.append((uint8_t)5); - Address(rules[i].v.zt).appendTo(tmp); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - tmp.append((uint8_t)2); - tmp.append((uint16_t)rules[i].v.vlanId); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - tmp.append((uint8_t)1); - tmp.append((uint8_t)rules[i].v.vlanPcp); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - tmp.append((uint8_t)1); - tmp.append((uint8_t)rules[i].v.vlanDei); - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - tmp.append((uint8_t)2); - tmp.append((uint16_t)rules[i].v.etherType); - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - tmp.append((uint8_t)6); - tmp.append(rules[i].v.mac,6); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - tmp.append((uint8_t)5); - tmp.append(&(rules[i].v.ipv4.ip),4); - tmp.append((uint8_t)rules[i].v.ipv4.mask); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - tmp.append((uint8_t)17); - tmp.append(rules[i].v.ipv6.ip,16); - tmp.append((uint8_t)rules[i].v.ipv6.mask); - break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - tmp.append((uint8_t)1); - tmp.append((uint8_t)rules[i].v.ipTos); - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - tmp.append((uint8_t)1); - tmp.append((uint8_t)rules[i].v.ipProtocol); - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - tmp.append((uint8_t)4); - tmp.append((uint16_t)rules[i].v.port[0]); - tmp.append((uint16_t)rules[i].v.port[1]); - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - tmp.append((uint8_t)16); - tmp.append((uint64_t)rules[i].v.characteristics[0]); - tmp.append((uint64_t)rules[i].v.characteristics[1]); - break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - tmp.append((uint8_t)4); - tmp.append((uint16_t)rules[i].v.frameSize[0]); - tmp.append((uint16_t)rules[i].v.frameSize[1]); - break; + tmp->clear(); + for(unsigned int i=0;irouteCount;++i) { + reinterpret_cast(&(this->routes[i].target))->serialize(*tmp); + reinterpret_cast(&(this->routes[i].via))->serialize(*tmp); + tmp->append((uint16_t)this->routes[i].flags); + tmp->append((uint16_t)this->routes[i].metric); } - } - if (tmp.size()) { - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_RULES,tmp)) return false; + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ROUTES,*tmp)) return false; + } + + tmp->clear(); + for(unsigned int i=0;istaticIpCount;++i) + this->staticIps[i].serialize(*tmp); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_STATIC_IPS,*tmp)) return false; + } + + if (this->ruleCount) { + tmp->clear(); + Capability::serializeRules(*tmp,rules,ruleCount); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_RULES,*tmp)) return false; + } + } + + delete tmp; + } catch ( ... ) { + delete tmp; + throw; } return true; @@ -230,26 +180,31 @@ bool NetworkConfig::toDictionary(Dictionary &d,b bool NetworkConfig::fromDictionary(const Dictionary &d) { - try { - Buffer tmp; - char tmp2[ZT_NETWORKCONFIG_DICT_CAPACITY]; + Buffer *tmp = new Buffer(); + try { memset(this,0,sizeof(NetworkConfig)); // Fields that are always present, new or old this->networkId = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,0); - if (!this->networkId) + if (!this->networkId) { + delete tmp; return false; + } this->timestamp = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,0); this->revision = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REVISION,0); this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0); - if (!this->issuedTo) + if (!this->issuedTo) { + delete tmp; return false; + } this->multicastLimit = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,0); d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name,sizeof(this->name)); if (d.getUI(ZT_NETWORKCONFIG_DICT_KEY_VERSION,0) < 6) { #ifdef ZT_SUPPORT_OLD_STYLE_NETCONF + char tmp2[1024]; + // Decode legacy fields if version is old if (d.getB(ZT_NETWORKCONFIG_DICT_KEY_ALLOW_PASSIVE_BRIDGING_OLD)) this->flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; @@ -305,6 +260,7 @@ bool NetworkConfig::fromDictionary(const Dictionaryflags = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,0); this->type = (ZT_VirtualNetworkType)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)ZT_NETWORK_TYPE_PRIVATE); - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_COM,tmp)) { - this->com.deserialize(tmp,0); + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_COM,*tmp)) + this->com.deserialize(*tmp,0); + + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_CAPABILITIES,*tmp)) { + try { + unsigned int p = 0; + while (p < tmp->size()) { + Capability cap; + p += cap.deserialize(*tmp,p); + this->capabilities[this->capabilityCount++] = cap; + } + } catch ( ... ) {} + std::sort(&(this->capabilities[0]),&(this->capabilities[this->capabilityCount])); + } + + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_TAGS,*tmp)) { + try { + unsigned int p = 0; + while (p < tmp->size()) { + Tag tag; + p += tag.deserialize(*tmp,p); + this->tags[this->tagCount++] = tag; + } + } catch ( ... ) {} + std::sort(&(this->tags[0]),&(this->tags[this->tagCount])); } - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_SPECIALISTS,tmp)) { + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_SPECIALISTS,*tmp)) { unsigned int p = 0; - while (((p + 8) <= tmp.size())&&(specialistCount < ZT_MAX_NETWORK_SPECIALISTS)) { - this->specialists[this->specialistCount++] = tmp.at(p); + while (((p + 8) <= tmp->size())&&(specialistCount < ZT_MAX_NETWORK_SPECIALISTS)) { + this->specialists[this->specialistCount++] = tmp->at(p); p += 8; } } - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_ROUTES,tmp)) { + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_ROUTES,*tmp)) { unsigned int p = 0; - while ((p < tmp.size())&&(routeCount < ZT_MAX_NETWORK_ROUTES)) { - p += reinterpret_cast(&(this->routes[this->routeCount].target))->deserialize(tmp,p); - p += reinterpret_cast(&(this->routes[this->routeCount].via))->deserialize(tmp,p); - this->routes[this->routeCount].flags = tmp.at(p); p += 2; - this->routes[this->routeCount].metric = tmp.at(p); p += 2; + while ((p < tmp->size())&&(routeCount < ZT_MAX_NETWORK_ROUTES)) { + p += reinterpret_cast(&(this->routes[this->routeCount].target))->deserialize(*tmp,p); + p += reinterpret_cast(&(this->routes[this->routeCount].via))->deserialize(*tmp,p); + this->routes[this->routeCount].flags = tmp->at(p); p += 2; + this->routes[this->routeCount].metric = tmp->at(p); p += 2; ++this->routeCount; } } - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_STATIC_IPS,tmp)) { + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_STATIC_IPS,*tmp)) { unsigned int p = 0; - while ((p < tmp.size())&&(staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - p += this->staticIps[this->staticIpCount++].deserialize(tmp,p); + while ((p < tmp->size())&&(staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + p += this->staticIps[this->staticIpCount++].deserialize(*tmp,p); } } - if (d.get(ZT_NETWORKCONFIG_DICT_KEY_RULES,tmp)) { + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_RULES,*tmp)) { + this->ruleCount = 0; unsigned int p = 0; - while ((p < tmp.size())&&(ruleCount < ZT_MAX_NETWORK_RULES)) { - rules[ruleCount].t = (uint8_t)tmp[p++]; - unsigned int fieldLen = (unsigned int)tmp[p++]; - switch((ZT_VirtualNetworkRuleType)(rules[ruleCount].t & 0x7f)) { - default: - break; - case ZT_NETWORK_RULE_ACTION_TEE: - case ZT_NETWORK_RULE_ACTION_REDIRECT: - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - rules[ruleCount].v.zt = Address(tmp.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - rules[ruleCount].v.vlanId = tmp.at(p); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - rules[ruleCount].v.vlanPcp = (uint8_t)tmp[p]; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - rules[ruleCount].v.vlanDei = (uint8_t)tmp[p]; - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - rules[ruleCount].v.etherType = tmp.at(p); - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - memcpy(rules[ruleCount].v.mac,tmp.field(p,6),6); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - memcpy(&(rules[ruleCount].v.ipv4.ip),tmp.field(p,4),4); - rules[ruleCount].v.ipv4.mask = (uint8_t)tmp[p + 4]; - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - memcpy(rules[ruleCount].v.ipv6.ip,tmp.field(p,16),16); - rules[ruleCount].v.ipv6.mask = (uint8_t)tmp[p + 16]; - break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - rules[ruleCount].v.ipTos = (uint8_t)tmp[p]; - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - rules[ruleCount].v.ipProtocol = (uint8_t)tmp[p]; - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - rules[ruleCount].v.port[0] = tmp.at(p); - rules[ruleCount].v.port[1] = tmp.at(p + 2); - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - rules[ruleCount].v.characteristics[0] = tmp.at(p); - rules[ruleCount].v.characteristics[1] = tmp.at(p + 8); - break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - rules[ruleCount].v.frameSize[0] = tmp.at(p); - rules[ruleCount].v.frameSize[0] = tmp.at(p + 2); - break; - } - p += fieldLen; - ++ruleCount; - } + Capability::deserializeRules(*tmp,p,this->rules,this->ruleCount,ZT_MAX_NETWORK_RULES); } } @@ -412,8 +332,10 @@ bool NetworkConfig::fromDictionary(const Dictionary &metaData, + const Dictionary &metaData, NetworkConfig &nc) = 0; }; diff --git a/node/Tag.hpp b/node/Tag.hpp index b4bc63c4..bb019474 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -117,11 +117,14 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + // These are the same between Tag and Capability b.append(_nwid); b.append(_ts); b.append(_expiration); b.append(_id); + b.append(_value); + _issuedTo.appendTo(b); _signedBy.appendTo(b); if (!forSign) { @@ -129,6 +132,7 @@ public: b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); // length of signature b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); } + b.append((uint16_t)0); // length of additional fields, currently 0 if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); @@ -139,11 +143,14 @@ public: { unsigned int p = startAt; + // These are the same between Tag and Capability _nwid = b.template at(p); p += 8; _ts = b.template at(p); p += 8; _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; + _value = b.template at(p); p += 4; + _issuedTo.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (b[p++] != 1) -- cgit v1.2.3 From bcd05fbdfa7e340ef4df962773bb7c32cf5013c2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 09:34:13 -0700 Subject: Chunking of network config replies. --- node/Dictionary.hpp | 34 ++++++++++++++++++++++++++++++++++ node/IncomingPacket.cpp | 48 +++++++++++++++++++++++++++++------------------- node/NetworkConfig.hpp | 2 ++ node/Packet.hpp | 14 +++++--------- 4 files changed, 70 insertions(+), 28 deletions(-) (limited to 'node') diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 59fc4bbf..5d453fd9 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -23,6 +23,7 @@ #include "Utils.hpp" #include "Buffer.hpp" #include "Address.hpp" +#include "C25519.hpp" #include @@ -443,6 +444,39 @@ public: return found; } + /** + * Sign this Dictionary, replacing any previous signature + * + * @param sigKey Key to use for signature in dictionary + * @param kp Key pair to sign with + */ + inline void wrapWithSignature(const char *sigKey,const C25519::Pair &kp) + { + this->erase(sigKey); + C25519::Signature sig(C25519::sign(kp,this->data(),this->sizeBytes())); + this->add(sigKey,sig.data,ZT_C25519_SIGNATURE_LEN); + } + + /** + * Verify signature (and erase signature key) + * + * This erases this Dictionary's signature key (if present) and verifies + * the signature. The key is erased to render the Dictionary into the + * original unsigned form it was signed in for verification purposes. + * + * @param sigKey Key to use for signature in dictionary + * @param pk Public key to check against + * @return True if signature was present and valid + */ + inline bool unwrapAndVerify(const char *sigKey,const C25519::Public &pk) + { + char sig[ZT_C25519_SIGNATURE_LEN+1]; + if (this->get(sigKey,sig,sizeof(sig)) != ZT_C25519_SIGNATURE_LEN) + return false; + this->erase(sigKey); + return C25519::verify(pk,this->data(),this->sizeBytes(),sig); + } + /** * @return Dictionary data as a 0-terminated C-string */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index fae689d1..147f54da 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -719,35 +719,46 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); const Dictionary metaData(metaDataBytes,metaDataLength); - //const uint64_t haveRevision = ((ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength + 8) <= size()) ? at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength) : 0ULL; - const unsigned int h = hops(); - const uint64_t pid = packetId(); - peer->received(_localAddress,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP); + const uint64_t requestPacketId = packetId(); + peer->received(_localAddress,_remoteAddress,h,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP); if (RR->localNetworkController) { NetworkConfig netconf; switch(RR->localNetworkController->doNetworkConfigRequest((h > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,netconf)) { case NetworkController::NETCONF_QUERY_OK: { - Dictionary dconf; - if (netconf.toDictionary(dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(pid); - outp.append(nwid); - const unsigned int dlen = dconf.sizeBytes(); - outp.append((uint16_t)dlen); - outp.append((const void *)dconf.data(),dlen); - outp.compress(); - RR->sw->send(outp,true,0); + Dictionary *dconf = new Dictionary(); + try { + if (netconf.toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { + const unsigned int totalSize = dconf->sizeBytes(); + unsigned int chunkPtr = 0; + while (chunkPtr < totalSize) { + const unsigned int chunkLen = std::min(totalSize - chunkPtr,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PROTO_MIN_PACKET_LENGTH + 32))); + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + outp.append(nwid); + outp.append((uint16_t)chunkLen); + outp.append((const void *)(dconf->data() + chunkPtr),chunkLen); + outp.append((uint32_t)totalSize); + outp.append((uint32_t)chunkPtr); + outp.compress(); + RR->sw->send(outp,true,0); + chunkPtr += chunkLen; + } + } + delete dconf; + } catch ( ... ) { + delete dconf; + throw; } } break; case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(pid); + outp.append(requestPacketId); outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); outp.append(nwid); outp.armor(peer->key(),true); @@ -757,7 +768,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons case NetworkController::NETCONF_QUERY_ACCESS_DENIED: { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(pid); + outp.append(requestPacketId); outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); outp.append(nwid); outp.armor(peer->key(),true); @@ -765,7 +776,6 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons } break; case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR: - // TRACE("NETWORK_CONFIG_REQUEST failed: internal error: %s",netconf.get("error","(unknown)").c_str()); break; case NetworkController::NETCONF_QUERY_IGNORE: @@ -779,7 +789,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons } else { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(pid); + outp.append(requestPacketId); outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); outp.append(nwid); outp.armor(peer->key(),true); diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 907da936..3682c466 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -135,6 +135,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_CAPABILITIES "CAP" // tags (binary blobs) #define ZT_NETWORKCONFIG_DICT_KEY_TAGS "TAG" +// curve25519 signature +#define ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE "C25519" // Legacy fields -- these are obsoleted but are included when older clients query diff --git a/node/Packet.hpp b/node/Packet.hpp index dce9f208..9d4c8289 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -724,7 +724,7 @@ public: * <[8] 64-bit network ID> * <[2] 16-bit length of request meta-data dictionary> * <[...] string-serialized request meta-data> - * [<[8] 64-bit timestamp of netconf we currently have>] + * <[8] 64-bit timestamp of netconf we currently have> * * This message requests network configuration from a node capable of * providing it. If the optional revision is included, a response is @@ -732,14 +732,10 @@ public: * * OK response payload: * <[8] 64-bit network ID> - * <[2] 16-bit length of network configuration dictionary field> - * <[...] network configuration dictionary (or fragment)> - * [<[4] 32-bit total length of assembled dictionary>] - * [<[4] 32-bit index of fragment in this reply>] - * - * Fields after the dictionary are extensions to support multipart - * sending of large network configs. If they are not present the - * sent config must be assumed to be whole. + * <[2] 16-bit length of network configuration dictionary chunk> + * <[...] network configuration dictionary (may be incomplete)> + * <[4] 32-bit total length of assembled dictionary> + * <[4] 32-bit index of chunk in this reply> * * ERROR response payload: * <[8] 64-bit network ID> -- cgit v1.2.3 From 4d498b3765695f1b82a2448f0e8efe698b33667d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 13:14:38 -0700 Subject: Handling of multi-part chunked network configs on the inbound side. --- node/Identity.hpp | 5 +++++ node/IncomingPacket.cpp | 16 ++++++++-------- node/Network.cpp | 48 +++++++++++++++++++++++++++++++++++++++++++++++- node/Network.hpp | 18 ++++++++++++++++++ node/NetworkConfig.cpp | 12 +++++++++++- node/NetworkConfig.hpp | 6 ++++-- 6 files changed, 93 insertions(+), 12 deletions(-) (limited to 'node') diff --git a/node/Identity.hpp b/node/Identity.hpp index 4aa93b87..ef7f2d77 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -282,6 +282,11 @@ public: bool fromString(const char *str); inline bool fromString(const std::string &str) { return fromString(str.c_str()); } + /** + * @return C25519 public key + */ + inline const C25519::Public &publicKey() const { return _publicKey; } + /** * @return True if this identity contains something */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 147f54da..e25cb058 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -402,15 +402,15 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_NETWORK_CONFIG_REQUEST: { const SharedPtr nw(RR->node->network(at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID))); if ((nw)&&(nw->controller() == peer->address())) { - const unsigned int nclen = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN); - if (nclen) { - Dictionary dconf((const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,nclen),nclen); - NetworkConfig nconf; - if (nconf.fromDictionary(dconf)) { - nw->setConfiguration(nconf,true); - TRACE("got network configuration for network %.16llx from %s",(unsigned long long)nw->id(),source().toString().c_str()); - } + const unsigned int chunkLen = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN); + const void *chunkData = field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,chunkLen); + unsigned int chunkIndex = 0; + unsigned int totalSize = chunkLen; + if ((ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen) < size()) { + totalSize = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen); + chunkIndex = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen + 4); } + nw->handleInboundConfigChunk(inRePacketId,chunkData,chunkLen,chunkIndex,totalSize); } } break; diff --git a/node/Network.cpp b/node/Network.cpp index 0fbdf5ba..b84756aa 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -343,6 +343,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _id(nwid), _mac(renv->identity.address(),nwid), _portInitialized(false), + _inboundConfigPacketId(0), _lastConfigUpdate(0), _destroyed(false), _netconfFailure(NETCONF_FAILURE_NONE), @@ -364,7 +365,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : std::string conf(RR->node->dataStoreGet(confn)); if (conf.length()) { dconf->load(conf.c_str()); - if (nconf->fromDictionary(*dconf)) { + if (nconf->fromDictionary(Identity(),*dconf)) { this->setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; @@ -589,6 +590,47 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) return 0; } +void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize) +{ + std::string newConfig; + if ((_inboundConfigPacketId == inRePacketId)&&(totalSize < ZT_NETWORKCONFIG_DICT_CAPACITY)&&((chunkIndex + chunkSize) < totalSize)) { + Mutex::Lock _l(_lock); + TRACE("got %u bytes at position %u of network config request %.16llx, total expected length %u",chunkSize,chunkIndex,inRePacketId,totalSize); + _inboundConfigChunks[chunkIndex].append((const char *)data,chunkSize); + unsigned int totalWeHave = 0; + for(std::map::iterator c(_inboundConfigChunks.begin());c!=_inboundConfigChunks.end();++c) + totalWeHave += (unsigned int)c->second.length(); + if (totalWeHave == totalSize) { + TRACE("have all chunks for network config request %.16llx, assembling...",inRePacketId); + for(std::map::iterator c(_inboundConfigChunks.begin());c!=_inboundConfigChunks.end();++c) + newConfig.append(c->second); + _inboundConfigPacketId = 0; + _inboundConfigChunks.clear(); + } else if (totalWeHave > totalSize) { + _inboundConfigPacketId = 0; + _inboundConfigChunks.clear(); + } + } + + if (newConfig.length() > 0) { + if (newConfig.length() < ZT_NETWORKCONFIG_DICT_CAPACITY) { + Dictionary *dict = new Dictionary(newConfig.c_str()); + try { + Identity controllerId(RR->topology->getIdentity(this->controller())); + if (controllerId) { + NetworkConfig nc; + if (nc.fromDictionary(controllerId,*dict)) + this->setConfiguration(nc,true); + } + delete dict; + } catch ( ... ) { + delete dict; + throw; + } + } + } +} + void Network::requestConfiguration() { if (_id == ZT_TEST_NETWORK_ID) // pseudo-network-ID, uses locally generated static config @@ -637,6 +679,10 @@ void Network::requestConfiguration() outp.append((_config) ? (uint64_t)_config.revision : (uint64_t)0); outp.compress(); RR->sw->send(outp,true,0); + + // Expect replies with this in-re packet ID + _inboundConfigPacketId = outp.packetId(); + _inboundConfigChunks.clear(); } void Network::clean() diff --git a/node/Network.hpp b/node/Network.hpp index 16f07163..d13918cf 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -214,6 +214,21 @@ public: */ int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); + /** + * Handle an inbound network config chunk + * + * Only chunks whose inRePacketId matches the packet ID of the last request + * are handled. If this chunk completes the config, it is decoded and + * setConfiguration() is called. + * + * @param inRePacketId In-re packet ID from OK(NETWORK_CONFIG_REQUEST) + * @param data Chunk data + * @param chunkSize Size of data[] + * @param chunkIndex Index of chunk in full config + * @param totalSize Total size of network config + */ + void handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize); + /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this */ @@ -411,6 +426,9 @@ private: Hashtable< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups that seem to be behind us and when we last saw them (if we are a bridge) Hashtable< MAC,Address > _remoteBridgeRoutes; // remote addresses where given MACs are reachable (for tracking devices behind remote bridges) + uint64_t _inboundConfigPacketId; + std::map _inboundConfigChunks; + NetworkConfig _config; volatile uint64_t _lastConfigUpdate; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 07e9bd4f..a4fddf40 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -178,8 +178,18 @@ bool NetworkConfig::toDictionary(Dictionary &d,b return true; } -bool NetworkConfig::fromDictionary(const Dictionary &d) +bool NetworkConfig::fromDictionary(const Identity &controllerId,Dictionary &d) { + if ((d.contains(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE))&&(controllerId)) { + // FIXME: right now signature are optional since network configs are only + // accepted directly from the controller and the protocol already guarantees + // the sender. In the future these might be made non-optional once old + // controllers that do not sign are gone and if we ever support peer caching + // of network configs. + if (!d.unwrapAndVerify(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE,controllerId.publicKey())) + return false; + } + Buffer *tmp = new Buffer(); try { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 3682c466..18244ec9 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -38,6 +38,7 @@ #include "Capability.hpp" #include "Tag.hpp" #include "Dictionary.hpp" +#include "Identity.hpp" /** * Flag: allow passive bridging (experimental) @@ -239,10 +240,11 @@ public: /** * Read this network config from a dictionary * - * @param d Dictionary + * @param controllerId Controller identity for verification of any signature or NULL identity to skip + * @param d Dictionary (non-const since it might be modified during parse, should not be used after call) * @return True if dictionary was valid and network config successfully initialized */ - bool fromDictionary(const Dictionary &d); + bool fromDictionary(const Identity &controllerId,Dictionary &d); /** * @return True if passive bridging is allowed (experimental) -- cgit v1.2.3 From 774c7e0ea57784fb39e9194246ca0b927637f8ba Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 13:52:08 -0700 Subject: Put CONFIG_REFRESH back. --- node/IncomingPacket.cpp | 136 ++++++++++++++++++++++++++++-------------------- node/IncomingPacket.hpp | 1 + 2 files changed, 80 insertions(+), 57 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e25cb058..41a9aeea 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -102,6 +102,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); + case Packet::VERB_NETWORK_CONFIG_REFRESH: return _doNETWORK_CONFIG_REFRESH(RR,peer); case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); @@ -724,67 +725,70 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons peer->received(_localAddress,_remoteAddress,h,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP); if (RR->localNetworkController) { - NetworkConfig netconf; - switch(RR->localNetworkController->doNetworkConfigRequest((h > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,netconf)) { - - case NetworkController::NETCONF_QUERY_OK: { - Dictionary *dconf = new Dictionary(); - try { - if (netconf.toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { - const unsigned int totalSize = dconf->sizeBytes(); - unsigned int chunkPtr = 0; - while (chunkPtr < totalSize) { - const unsigned int chunkLen = std::min(totalSize - chunkPtr,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PROTO_MIN_PACKET_LENGTH + 32))); - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append(nwid); - outp.append((uint16_t)chunkLen); - outp.append((const void *)(dconf->data() + chunkPtr),chunkLen); - outp.append((uint32_t)totalSize); - outp.append((uint32_t)chunkPtr); - outp.compress(); - RR->sw->send(outp,true,0); - chunkPtr += chunkLen; + NetworkConfig *netconf = new NetworkConfig(); + try { + switch(RR->localNetworkController->doNetworkConfigRequest((h > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,*netconf)) { + + case NetworkController::NETCONF_QUERY_OK: { + Dictionary *dconf = new Dictionary(); + try { + if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { + const unsigned int totalSize = dconf->sizeBytes(); + unsigned int chunkIndex = 0; + while (chunkIndex < totalSize) { + const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PROTO_MIN_PACKET_LENGTH + 32))); + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + outp.append(nwid); + outp.append((uint16_t)chunkLen); + outp.append((const void *)(dconf->data() + chunkIndex),chunkLen); + outp.append((uint32_t)totalSize); + outp.append((uint32_t)chunkIndex); + outp.compress(); + RR->sw->send(outp,true,0); + chunkIndex += chunkLen; + } } + delete dconf; + } catch ( ... ) { + delete dconf; + throw; } - delete dconf; - } catch ( ... ) { - delete dconf; - throw; - } - } break; - - case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); - outp.append(nwid); - outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); - } break; - - case NetworkController::NETCONF_QUERY_ACCESS_DENIED: { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); - outp.append(nwid); - outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); - } break; - - case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR: - break; - - case NetworkController::NETCONF_QUERY_IGNORE: - break; + } break; - default: - TRACE("NETWORK_CONFIG_REQUEST failed: invalid return value from NetworkController::doNetworkConfigRequest()"); - break; + case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); + outp.append(nwid); + outp.armor(peer->key(),true); + RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + } break; + case NetworkController::NETCONF_QUERY_ACCESS_DENIED: { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); + outp.append(nwid); + outp.armor(peer->key(),true); + RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + } break; + + case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR: + break; + case NetworkController::NETCONF_QUERY_IGNORE: + break; + default: + TRACE("NETWORK_CONFIG_REQUEST failed: invalid return value from NetworkController::doNetworkConfigRequest()"); + break; + } + delete netconf; + } catch ( ... ) { + delete netconf; + throw; } } else { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); @@ -801,6 +805,24 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons return true; } +bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer) +{ + try { + unsigned int p = ZT_PACKET_IDX_PAYLOAD; + while ((p + 8) <= size()) { + const uint64_t nwid = at(p); p += 8; + if (Network::controllerFor(nwid) == peer->address()) { + SharedPtr network(RR->node->network(nwid)); + if (network) + network->requestConfiguration(); + } + } + } catch ( ... ) { + TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + } + return true; +} + bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 558dfaa2..303ac5f8 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -165,6 +165,7 @@ private: bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer); + bool _doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); -- cgit v1.2.3 From dee7f75f7e0ed33997299fdbab0de4b63a9a69d6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 14:46:11 -0700 Subject: Minor cleanup. --- node/IncomingPacket.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 41a9aeea..b6ec2d3f 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -150,7 +150,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr break; case Packet::ERROR_IDENTITY_COLLISION: - if (RR->topology->isUpstream(peer->identity())) + if (RR->topology->isRoot(peer->identity())) RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; @@ -421,7 +421,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); - TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,mg.toString().c_str(),size()); + //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); } break; @@ -720,14 +720,14 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); const Dictionary metaData(metaDataBytes,metaDataLength); - const unsigned int h = hops(); + const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - peer->received(_localAddress,_remoteAddress,h,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP); if (RR->localNetworkController) { NetworkConfig *netconf = new NetworkConfig(); try { - switch(RR->localNetworkController->doNetworkConfigRequest((h > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,*netconf)) { + switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,*netconf)) { case NetworkController::NETCONF_QUERY_OK: { Dictionary *dconf = new Dictionary(); @@ -736,7 +736,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const unsigned int totalSize = dconf->sizeBytes(); unsigned int chunkIndex = 0; while (chunkIndex < totalSize) { - const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PROTO_MIN_PACKET_LENGTH + 32))); + const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PACKET_IDX_PAYLOAD + 32))); Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); outp.append(requestPacketId); -- cgit v1.2.3 From dbf3e6c3c9214d1b584da18f82743aa7c301ad3e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 15:01:46 -0700 Subject: Dead code removal. --- node/MulticastGroup.hpp | 26 -------------------------- 1 file changed, 26 deletions(-) (limited to 'node') diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index dbf38998..be4e8084 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -60,16 +60,6 @@ public: { } - MulticastGroup(const char *s) - { - fromString(s); - } - - MulticastGroup(const std::string &s) - { - fromString(s.c_str()); - } - /** * Derive the multicast group used for address resolution (ARP/NDP) for an IP * @@ -106,22 +96,6 @@ public: return std::string(buf); } - /** - * Parse a human-readable multicast group - * - * @param s Multicast group in hex MAC/ADI format - */ - inline void fromString(const char *s) - { - char hex[17]; - unsigned int hexlen = 0; - while ((*s)&&(*s != '/')&&(hexlen < (sizeof(hex) - 1))) - hex[hexlen++] = *s; - hex[hexlen] = (char)0; - _mac.fromString(hex); - _adi = (*s == '/') ? (uint32_t)Utils::hexStrToULong(s + 1) : (uint32_t)0; - } - /** * @return Multicast address */ -- cgit v1.2.3 From e1310a764a39d0ed1f29f213c6e75c4e2d7a8aba Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 15:45:26 -0700 Subject: More cleanup and removal of cruft due to obsolete network-specific relays (will be replaced with federation stuff). --- node/IncomingPacket.cpp | 16 ++++++---- node/Membership.cpp | 2 +- node/Multicaster.cpp | 2 +- node/Network.cpp | 8 ++--- node/Node.cpp | 4 +-- node/OutboundMulticast.cpp | 2 +- node/Peer.cpp | 8 ++--- node/Peer.hpp | 3 +- node/SelfAwareness.cpp | 2 +- node/Switch.cpp | 76 ++++++---------------------------------------- node/Switch.hpp | 42 ++----------------------- 11 files changed, 37 insertions(+), 128 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b6ec2d3f..df224c19 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -510,9 +510,13 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP); const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); - if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) - RR->sw->rendezvous(withPeer,_localAddress,atAddr); + TRACE("RENDEZVOUS from %s says %s might be at %s, attempting to contact",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) { + const uint64_t now = RR->node->now(); + peer->sendHELLO(_localAddress,atAddr,now,2); // send low-TTL packet to 'open' local NAT(s) + if (!peer->pushDirectPaths(_localAddress,atAddr,now,true)) + peer->sendHELLO(_localAddress,atAddr,now); + } } else { TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -746,7 +750,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append((uint32_t)totalSize); outp.append((uint32_t)chunkIndex); outp.compress(); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); chunkIndex += chunkLen; } } @@ -1139,7 +1143,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt nextHop[h].appendTo(outp); nextHopBestPathAddress[h].serialize(outp); // appends 0 if null InetAddress } - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); } // If there are next hops, forward the test along through the graph @@ -1154,7 +1158,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt if (RR->identity.address() != nextHop[h]) { // next hops that loop back to the current hop are not valid outp.newInitializationVector(); outp.setDestination(nextHop[h]); - RR->sw->send(outp,true,originatorCredentialNetworkId); + RR->sw->send(outp,true); } } } diff --git a/node/Membership.cpp b/node/Membership.cpp index e12bce3c..dbba7f0d 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -69,7 +69,7 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint outp.append((uint8_t)0x00); outp.append(capsAndTags.data(),capsAndTags.size()); outp.compress(); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); _lastPushedCom = now; return true; } diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 9e583e34..aeee0a85 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -240,7 +240,7 @@ void Multicaster::send( mg.mac().appendTo(outp); outp.append((uint32_t)mg.adi()); outp.append((uint32_t)gatherLimit); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); } gatherLimit = 0; } diff --git a/node/Network.cpp b/node/Network.cpp index b84756aa..e098c1fd 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -127,7 +127,7 @@ static int _doZtFilter( outp.append((uint16_t)etherType); outp.append(frameData,frameLen); outp.compress(); - RR->sw->send(outp,true,nwid); + RR->sw->send(outp,true); if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { return -1; // match, drop packet (we redirected it) @@ -678,7 +678,7 @@ void Network::requestConfiguration() outp.append((const void *)rmd.data(),rmdSize); outp.append((_config) ? (uint64_t)_config.revision : (uint64_t)0); outp.compress(); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); // Expect replies with this in-re packet ID _inboundConfigPacketId = outp.packetId(); @@ -894,7 +894,7 @@ void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std:: for(std::vector::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { if ((outp.size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { outp.compress(); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); outp.reset(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); } @@ -906,7 +906,7 @@ void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std:: if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) { outp.compress(); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); } } diff --git a/node/Node.cpp b/node/Node.cpp index f04559db..4da79347 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -237,7 +237,7 @@ public: // way whatsoever. This will e.g. find network preferred relays that lack // stable endpoints by using root servers. Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); @@ -520,7 +520,7 @@ ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback) for(unsigned int a=0;ahops[0].breadth;++a) { outp.newInitializationVector(); outp.setDestination(Address(test->hops[0].addresses[a])); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); } } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index a5856164..c9952927 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -90,7 +90,7 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toA //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr); - RR->sw->send(_packet,true,_nwid); + RR->sw->send(_packet,true); } } diff --git a/node/Peer.cpp b/node/Peer.cpp index 89dce570..77e1d0b5 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -241,7 +241,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) return false; } -bool Peer::pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now,bool force,bool includePrivatePaths) +bool Peer::pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now,bool force) { #ifdef ZT_ENABLE_CLUSTER // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection @@ -258,10 +258,8 @@ bool Peer::pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAdd std::vector pathsToPush; std::vector dps(RR->node->directPaths()); - for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) { - if ((includePrivatePaths)||(i->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) - pathsToPush.push_back(*i); - } + for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) + pathsToPush.push_back(*i); std::vector sym(RR->sa->getSymmetricNatPredictions()); for(unsigned long i=0,added=0;i >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) { if ((*p)->activelyTransferringFrames(now)) { Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP); - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); } } } else { diff --git a/node/Switch.cpp b/node/Switch.cpp index 167c7928..37daff27 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -476,14 +476,14 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append((uint16_t)etherType); outp.append(data,len); outp.compress(); - send(outp,true,network->id()); + send(outp,true); } else { Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME); outp.append(network->id()); outp.append((uint16_t)etherType); outp.append(data,len); outp.compress(); - send(outp,true,network->id()); + send(outp,true); } //TRACE("%.16llx: UNICAST: %s -> %s etherType==%s(%.4x) vlanId==%u len==%u fromBridged==%d includeCom==%d",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),etherType,vlanId,len,(int)fromBridged,(int)includeCom); @@ -536,23 +536,21 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append((uint16_t)etherType); outp.append(data,len); outp.compress(); - send(outp,true,network->id()); + send(outp,true); } } } -void Switch::send(const Packet &packet,bool encrypt,uint64_t nwid) +void Switch::send(const Packet &packet,bool encrypt) { if (packet.destination() == RR->identity.address()) { TRACE("BUG: caught attempt to send() to self, ignored"); return; } - //TRACE(">> %s to %s (%u bytes, encrypt==%d, nwid==%.16llx)",Packet::verbString(packet.verb()),packet.destination().toString().c_str(),packet.size(),(int)encrypt,nwid); - - if (!_trySend(packet,encrypt,nwid)) { + if (!_trySend(packet,encrypt)) { Mutex::Lock _l(_txQueue_m); - _txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt,nwid)); + _txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt)); } } @@ -625,17 +623,6 @@ bool Switch::unite(const Address &p1,const Address &p2) return true; } -void Switch::rendezvous(const SharedPtr &peer,const InetAddress &localAddr,const InetAddress &atAddr) -{ - TRACE("sending NAT-t message to %s(%s)",peer->address().toString().c_str(),atAddr.toString().c_str()); - const uint64_t now = RR->node->now(); - peer->sendHELLO(localAddr,atAddr,now,2); // first attempt: send low-TTL packet to 'open' local NAT - { - Mutex::Lock _l(_contactQueue_m); - _contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,localAddr,atAddr)); - } -} - void Switch::requestWhois(const Address &addr) { bool inserted = false; @@ -676,7 +663,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { if (txi->dest == peer->address()) { - if (_trySend(txi->packet,txi->encrypt,txi->nwid)) + if (_trySend(txi->packet,txi->encrypt)) _txQueue.erase(txi++); else ++txi; } else ++txi; @@ -688,42 +675,6 @@ unsigned long Switch::doTimerTasks(uint64_t now) { unsigned long nextDelay = 0xffffffff; // ceiling delay, caller will cap to minimum - { // Iterate through NAT traversal strategies for entries in contact queue - Mutex::Lock _l(_contactQueue_m); - for(std::list::iterator qi(_contactQueue.begin());qi!=_contactQueue.end();) { - if (now >= qi->fireAtTime) { - if (!qi->peer->pushDirectPaths(qi->localAddr,qi->inaddr,now,true,false)) - qi->peer->sendHELLO(qi->localAddr,qi->inaddr,now); - _contactQueue.erase(qi++); - continue; - /* Old symmetric NAT buster code, obsoleted by port prediction alg in SelfAwareness but left around for now in case we revert - if (qi->strategyIteration == 0) { - // First strategy: send packet directly to destination - qi->peer->sendHELLO(qi->localAddr,qi->inaddr,now); - } else if (qi->strategyIteration <= 3) { - // Strategies 1-3: try escalating ports for symmetric NATs that remap sequentially - InetAddress tmpaddr(qi->inaddr); - int p = (int)qi->inaddr.port() + qi->strategyIteration; - if (p > 65535) - p -= 64511; - tmpaddr.setPort((unsigned int)p); - qi->peer->sendHELLO(qi->localAddr,tmpaddr,now); - } else { - // All strategies tried, expire entry - _contactQueue.erase(qi++); - continue; - } - ++qi->strategyIteration; - qi->fireAtTime = now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY; - nextDelay = std::min(nextDelay,(unsigned long)ZT_NAT_T_TACTICAL_ESCALATION_DELAY); - */ - } else { - nextDelay = std::min(nextDelay,(unsigned long)(qi->fireAtTime - now)); - } - ++qi; // if qi was erased, loop will have continued before here - } - } - { // Retry outstanding WHOIS requests Mutex::Lock _l(_outstandingWhoisRequests_m); Hashtable< Address,WhoisRequest >::Iterator i(_outstandingWhoisRequests); @@ -751,7 +702,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) { // Time out TX queue packets that never got WHOIS lookups or other info. Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { - if (_trySend(txi->packet,txi->encrypt,txi->nwid)) + if (_trySend(txi->packet,txi->encrypt)) _txQueue.erase(txi++); else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { TRACE("TX %s -> %s timed out",txi->packet.source().toString().c_str(),txi->packet.destination().toString().c_str()); @@ -787,20 +738,13 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread return Address(); } -bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid) +bool Switch::_trySend(const Packet &packet,bool encrypt) { SharedPtr peer(RR->topology->getPeer(packet.destination())); if (peer) { const uint64_t now = RR->node->now(); - SharedPtr network; - if (nwid) { - network = RR->node->network(nwid); - if ((!network)||(!network->hasConfig())) - return false; // we probably just left this network, let its packets die - } - Path *viaPath = peer->getBestPath(now); SharedPtr relay; @@ -811,7 +755,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid) } if (relay) { - peer->pushDirectPaths(viaPath->localAddress(),viaPath->address(),now,false,( (network)&&(network->isAllowed(peer)) )); + peer->pushDirectPaths(viaPath->localAddress(),viaPath->address(),now,false); viaPath->sent(now); } diff --git a/node/Switch.hpp b/node/Switch.hpp index ce4f00a1..7c903ef9 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -92,15 +92,10 @@ public: * Needless to say, the packet's source must be this node. Otherwise it * won't be encrypted right. (This is not used for relaying.) * - * The network ID should only be specified for frames and other actual - * network traffic. Other traffic such as controller requests and regular - * protocol messages should specify zero. - * * @param packet Packet to send * @param encrypt Encrypt packet payload? (always true except for HELLO) - * @param nwid Related network ID or 0 if message is not in-network traffic */ - void send(const Packet &packet,bool encrypt,uint64_t nwid); + void send(const Packet &packet,bool encrypt); /** * Send RENDEZVOUS to two peers to permit them to directly connect @@ -113,15 +108,6 @@ public: */ bool unite(const Address &p1,const Address &p2); - /** - * Attempt NAT traversal to peer at a given physical address - * - * @param peer Peer to contact - * @param localAddr Local interface address - * @param atAddr Address of peer - */ - void rendezvous(const SharedPtr &peer,const InetAddress &localAddr,const InetAddress &atAddr); - /** * Request WHOIS on a given address * @@ -151,7 +137,7 @@ public: private: Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); - bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid); + bool _trySend(const Packet &packet,bool encrypt); const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; @@ -205,16 +191,14 @@ private: struct TXQueueEntry { TXQueueEntry() {} - TXQueueEntry(Address d,uint64_t ct,const Packet &p,bool enc,uint64_t nw) : + TXQueueEntry(Address d,uint64_t ct,const Packet &p,bool enc) : dest(d), creationTime(ct), - nwid(nw), packet(p), encrypt(enc) {} Address dest; uint64_t creationTime; - uint64_t nwid; Packet packet; // unencrypted/unMAC'd packet -- this is done at send time bool encrypt; }; @@ -241,26 +225,6 @@ private: }; Hashtable< _LastUniteKey,uint64_t > _lastUniteAttempt; // key is always sorted in ascending order, for set-like behavior Mutex _lastUniteAttempt_m; - - // Active attempts to contact remote peers, including state of multi-phase NAT traversal - struct ContactQueueEntry - { - ContactQueueEntry() {} - ContactQueueEntry(const SharedPtr &p,uint64_t ft,const InetAddress &laddr,const InetAddress &a) : - peer(p), - fireAtTime(ft), - inaddr(a), - localAddr(laddr), - strategyIteration(0) {} - - SharedPtr peer; - uint64_t fireAtTime; - InetAddress inaddr; - InetAddress localAddr; - unsigned int strategyIteration; - }; - std::list _contactQueue; - Mutex _contactQueue_m; }; } // namespace ZeroTier -- cgit v1.2.3 From 0b0cda2be4f93449ea490e8313f1f69175a49819 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 15:55:41 -0700 Subject: ZT_TRACE fix. --- node/Packet.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index eda60757..aadee00b 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -40,6 +40,7 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; case VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; + case VERB_NETWORK_CONFIG_REFRESH: return "NETWORK_CONFIG_REFRESH"; case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; -- cgit v1.2.3 From c9d7845fea15ffe0e09295aedba6389de1bcb59b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 Aug 2016 17:00:01 -0700 Subject: Minor bug fix and some instrumentation stuff for testing. --- node/IncomingPacket.cpp | 2 +- node/Network.cpp | 7 ++++++- node/Packet.hpp | 1 + node/SelfAwareness.cpp | 42 ++++++++++++++++++++++++++++++++++++------ node/SelfAwareness.hpp | 5 +++-- 5 files changed, 47 insertions(+), 10 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index df224c19..53f6b88a 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -84,7 +84,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } const Packet::Verb v = verb(); - //TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" diff --git a/node/Network.cpp b/node/Network.cpp index e098c1fd..b9a2ca1d 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -676,7 +676,12 @@ void Network::requestConfiguration() const unsigned int rmdSize = rmd.sizeBytes(); outp.append((uint16_t)rmdSize); outp.append((const void *)rmd.data(),rmdSize); - outp.append((_config) ? (uint64_t)_config.revision : (uint64_t)0); + if (_config) { + outp.append((uint64_t)_config.revision); + outp.append((uint64_t)_config.timestamp); + } else { + outp.append((unsigned char)0,16); + } outp.compress(); RR->sw->send(outp,true); diff --git a/node/Packet.hpp b/node/Packet.hpp index 9d4c8289..0524139d 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -724,6 +724,7 @@ public: * <[8] 64-bit network ID> * <[2] 16-bit length of request meta-data dictionary> * <[...] string-serialized request meta-data> + * <[8] 64-bit revision of netconf we currently have> * <[8] 64-bit timestamp of netconf we currently have> * * This message requests network configuration from a node capable of diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index a4fae3d5..05df53fe 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -79,9 +79,10 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc if ( (trusted) && ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) { // Changes to external surface reported by trusted peers causes path reset in this scope + TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str()); entry.mySurface = myPhysicalAddress; entry.ts = now; - TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str()); + entry.trusted = trusted; // Erase all entries in this scope that were not reported from this remote address to prevent 'thrashing' // due to multiple reports of endpoint change. @@ -113,6 +114,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc // Otherwise just update DB to use to determine external surface info entry.mySurface = myPhysicalAddress; entry.ts = now; + entry.trusted = trusted; } } @@ -148,22 +150,50 @@ std::vector SelfAwareness::getSymmetricNatPredictions() bool symmetric = false; { Mutex::Lock _l(_phy_m); + Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); PhySurfaceKey *k = (PhySurfaceKey *)0; PhySurfaceEntry *e = (PhySurfaceEntry *)0; + InetAddress lastTrustedSurface; while (i.next(k,e)) { if ((e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { std::set &s = surfaces[k->receivedOnLocalAddress]; - s.insert(e->mySurface); + + /* MINOR SECURITY FIX: + * + * If the surface was not reported by a trusted (upstream) peer, we do + * not use its report of our surface IP for symmetric NAT prediction. + * Otherwise a peer could poison our external surface cache and then + * use this to coax us into suggesting their IP as an endpoint. This + * in turn could allow them to relay traffic for us. They could not + * decrypt or otherwise mess with it, but they could DOS us or record + * meta-data without anything appearing amiss. + * + * So for surfaces reported by untrusted peers we use the IP reported + * by a trusted peer and then just use the port. + * + * As far as we know this has never been exploited. We discovered it + * because certain weird configurations, such as load balancers and + * gateways that do not preserve IP information, can coax a node into + * reporting back false surface information. */ + if (e->trusted) { + s.insert(e->mySurface); + lastTrustedSurface = e->mySurface; + } else if (lastTrustedSurface) { + InetAddress tmp(lastTrustedSurface); + tmp.setPort(e->mySurface.port()); + s.insert(tmp); + } + symmetric = symmetric||(s.size() > 1); } } } - // If we appear to be symmetrically NATed, generate and return extrapolations - // of those surfaces. Since PUSH_DIRECT_PATHS is sent multiple times, we - // probabilistically generate extrapolations of anywhere from +1 to +5 to - // increase the odds that it will work "eventually". + /* If we appear to be symmetrically NATed, generate and return extrapolations + * of those surfaces. Since PUSH_DIRECT_PATHS is sent multiple times, we + * probabilistically generate extrapolations of anywhere from +1 to +5 to + * increase the odds that it will work "eventually". */ if (symmetric) { std::vector r; for(std::map< InetAddress,std::set >::iterator si(surfaces.begin());si!=surfaces.end();++si) { diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index 06c264a9..c7bde87e 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -82,9 +82,10 @@ private: { InetAddress mySurface; uint64_t ts; + bool trusted; - PhySurfaceEntry() : mySurface(),ts(0) {} - PhySurfaceEntry(const InetAddress &a,const uint64_t t) : mySurface(a),ts(t) {} + PhySurfaceEntry() : mySurface(),ts(0),trusted(false) {} + PhySurfaceEntry(const InetAddress &a,const uint64_t t) : mySurface(a),ts(t),trusted(false) {} }; const RuntimeEnvironment *RR; -- cgit v1.2.3 From 81959f14afe8c236446c2fd5a3c30da1fbb942de Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 10 Aug 2016 10:28:54 -0700 Subject: Refactor and redesign symmetric NAT predictor. This is cleaner. --- node/InetAddress.hpp | 20 ++++++++- node/SelfAwareness.cpp | 112 +++++++++++++++++++++++-------------------------- 2 files changed, 72 insertions(+), 60 deletions(-) (limited to 'node') diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index e03deb71..b97e2ca6 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -356,7 +356,6 @@ struct InetAddress : public sockaddr_storage * @return pointer to raw address bytes or NULL if not available */ inline const void *rawIpData() const - throw() { switch(ss_family) { case AF_INET: return (const void *)&(reinterpret_cast(this)->sin_addr.s_addr); @@ -365,6 +364,25 @@ struct InetAddress : public sockaddr_storage } } + /** + * @return InetAddress containing only the IP portion of this address and a zero port, or NULL if not IPv4 or IPv6 + */ + inline InetAddress ipOnly() const + { + InetAddress r; + switch(ss_family) { + case AF_INET: + r.ss_family = AF_INET; + reinterpret_cast(&r)->sin_addr.s_addr = reinterpret_cast(this)->sin_addr.s_addr; + break; + case AF_INET6: + r.ss_family = AF_INET6; + memcpy(reinterpret_cast(&r)->sin6_addr.s6_addr,reinterpret_cast(this)->sin6_addr.s6_addr,16); + break; + } + return r; + } + /** * Performs an IP-only comparison or, if that is impossible, a memcmp() * diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 05df53fe..b9ab9d67 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -80,6 +80,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc if ( (trusted) && ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) { // Changes to external surface reported by trusted peers causes path reset in this scope TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str()); + entry.mySurface = myPhysicalAddress; entry.ts = now; entry.trusted = trusted; @@ -135,77 +136,70 @@ std::vector SelfAwareness::getSymmetricNatPredictions() /* This is based on ideas and strategies found here: * https://tools.ietf.org/html/draft-takeda-symmetric-nat-traversal-00 * - * In short: a great many symmetric NATs allocate ports sequentially. - * This is common on enterprise and carrier grade NATs as well as consumer - * devices. This code generates a list of "you might try this" addresses by - * extrapolating likely port assignments from currently known external - * global IPv4 surfaces. These can then be included in a PUSH_DIRECT_PATHS - * message to another peer, causing it to possibly try these addresses and - * bust our local symmetric NAT. It works often enough to be worth the - * extra bit of code and does no harm in cases where it fails. */ - - // Gather unique surfaces indexed by local received-on address and flag - // us as behind a symmetric NAT if there is more than one. - std::map< InetAddress,std::set > surfaces; + * For each IP address reported by a trusted (upstream) peer, we find + * the external port most recently reported by ANY peer for that IP. + * + * We only do any of this for global IPv4 addresses since private IPs + * and IPv6 are not going to have symmetric NAT. + * + * SECURITY NOTE: + * + * We never use IPs reported by non-trusted peers, since this could lead + * to a minor vulnerability whereby a peer could poison our cache with + * bad external surface reports via OK(HELLO) and then possibly coax us + * into suggesting their IP to other peers via PUSH_DIRECT_PATHS. This + * in turn could allow them to MITM flows. + * + * Since flows are encrypted and authenticated they could not actually + * read or modify traffic, but they could gather meta-data for forensics + * purpsoes or use this as a DOS attack vector. */ + + std::map< uint32_t,std::pair > maxPortByIp; + InetAddress theOneTrueSurface; bool symmetric = false; { Mutex::Lock _l(_phy_m); - Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); - PhySurfaceKey *k = (PhySurfaceKey *)0; - PhySurfaceEntry *e = (PhySurfaceEntry *)0; - InetAddress lastTrustedSurface; - while (i.next(k,e)) { - if ((e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { - std::set &s = surfaces[k->receivedOnLocalAddress]; - - /* MINOR SECURITY FIX: - * - * If the surface was not reported by a trusted (upstream) peer, we do - * not use its report of our surface IP for symmetric NAT prediction. - * Otherwise a peer could poison our external surface cache and then - * use this to coax us into suggesting their IP as an endpoint. This - * in turn could allow them to relay traffic for us. They could not - * decrypt or otherwise mess with it, but they could DOS us or record - * meta-data without anything appearing amiss. - * - * So for surfaces reported by untrusted peers we use the IP reported - * by a trusted peer and then just use the port. - * - * As far as we know this has never been exploited. We discovered it - * because certain weird configurations, such as load balancers and - * gateways that do not preserve IP information, can coax a node into - * reporting back false surface information. */ - if (e->trusted) { - s.insert(e->mySurface); - lastTrustedSurface = e->mySurface; - } else if (lastTrustedSurface) { - InetAddress tmp(lastTrustedSurface); - tmp.setPort(e->mySurface.port()); - s.insert(tmp); + { // First get IPs from only trusted peers, and perform basic NAT type characterization + Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); + PhySurfaceKey *k = (PhySurfaceKey *)0; + PhySurfaceEntry *e = (PhySurfaceEntry *)0; + while (i.next(k,e)) { + if ((e->trusted)&&(e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { + if (!theOneTrueSurface) + theOneTrueSurface = e->mySurface; + else if (theOneTrueSurface != e->mySurface) + symmetric = true; + maxPortByIp[reinterpret_cast(&(e->mySurface))->sin_addr.s_addr] = std::pair(e->ts,e->mySurface.port()); } + } + } - symmetric = symmetric||(s.size() > 1); + { // Then find max port per IP from a trusted peer + Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); + PhySurfaceKey *k = (PhySurfaceKey *)0; + PhySurfaceEntry *e = (PhySurfaceEntry *)0; + while (i.next(k,e)) { + if ((e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { + std::map< uint32_t,std::pair >::iterator mp(maxPortByIp.find(reinterpret_cast(&(e->mySurface))->sin_addr.s_addr)); + if ((mp != maxPortByIp.end())&&(mp->second.first < e->ts)) { + mp->second.first = e->ts; + mp->second.second = e->mySurface.port(); + } + } } } } - /* If we appear to be symmetrically NATed, generate and return extrapolations - * of those surfaces. Since PUSH_DIRECT_PATHS is sent multiple times, we - * probabilistically generate extrapolations of anywhere from +1 to +5 to - * increase the odds that it will work "eventually". */ if (symmetric) { std::vector r; - for(std::map< InetAddress,std::set >::iterator si(surfaces.begin());si!=surfaces.end();++si) { - for(std::set::iterator i(si->second.begin());i!=si->second.end();++i) { - InetAddress ipp(*i); - unsigned int p = ipp.port() + 1 + ((unsigned int)RR->node->prng() & 3); - if (p >= 65535) - p -= 64510; // NATs seldom use ports <=1024 so wrap to 1025 - ipp.setPort(p); - if ((si->second.count(ipp) == 0)&&(std::find(r.begin(),r.end(),ipp) == r.end())) { - r.push_back(ipp); - } + for(unsigned int k=1;k<=3;++k) { + for(std::map< uint32_t,std::pair >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) { + unsigned int p = i->second.second + k; + if (p > 65535) p -= 64511; + InetAddress pred(&(i->first),4,p); + if (std::find(r.begin(),r.end(),pred) == r.end()) + r.push_back(pred); } } return r; -- cgit v1.2.3 From d166b494ee4eee8f054f23508c1fbfac5a8bfc04 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 10 Aug 2016 13:41:22 -0700 Subject: Rule parse fix. --- node/Capability.hpp | 57 +++++++++++++++++++++++++------------------------ node/IncomingPacket.cpp | 51 +++++++++++++++++++++---------------------- node/Network.cpp | 38 ++++++++++++++++++++------------- 3 files changed, 78 insertions(+), 68 deletions(-) (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index fd6ae091..c129485d 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -176,7 +176,6 @@ public: template static inline void serializeRules(Buffer &b,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { - b.append((uint16_t)ruleCount); for(unsigned int i=0;i static inline void deserializeRules(const Buffer &b,unsigned int &p,ZT_VirtualNetworkRule *rules,unsigned int &ruleCount,const unsigned int maxRuleCount) { - ruleCount = b.template at(p); p += 2; - if (ruleCount > maxRuleCount) - throw std::runtime_error("rule count overflow"); - for(unsigned int i=0;i(p); + rules[ruleCount].v.vlanId = b.template at(p); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - rules[i].v.vlanPcp = (uint8_t)b[p]; + rules[ruleCount].v.vlanPcp = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - rules[i].v.vlanDei = (uint8_t)b[p]; + rules[ruleCount].v.vlanDei = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - rules[i].v.etherType = b.template at(p); + rules[ruleCount].v.etherType = b.template at(p); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: - memcpy(rules[i].v.mac,b.field(p,6),6); + memcpy(rules[ruleCount].v.mac,b.field(p,6),6); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - memcpy(&(rules[i].v.ipv4.ip),b.field(p,4),4); - rules[i].v.ipv4.mask = (uint8_t)b[p + 4]; + memcpy(&(rules[ruleCount].v.ipv4.ip),b.field(p,4),4); + rules[ruleCount].v.ipv4.mask = (uint8_t)b[p + 4]; break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - memcpy(rules[i].v.ipv6.ip,b.field(p,16),16); - rules[i].v.ipv6.mask = (uint8_t)b[p + 16]; + memcpy(rules[ruleCount].v.ipv6.ip,b.field(p,16),16); + rules[ruleCount].v.ipv6.mask = (uint8_t)b[p + 16]; break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - rules[i].v.ipTos = (uint8_t)b[p]; + rules[ruleCount].v.ipTos = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - rules[i].v.ipProtocol = (uint8_t)b[p]; + rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - rules[i].v.port[0] = b.template at(p); - rules[i].v.port[1] = b.template at(p + 2); + rules[ruleCount].v.port[0] = b.template at(p); + rules[ruleCount].v.port[1] = b.template at(p + 2); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - rules[i].v.characteristics[0] = b.template at(p); - rules[i].v.characteristics[1] = b.template at(p + 8); + rules[ruleCount].v.characteristics[0] = b.template at(p); + rules[ruleCount].v.characteristics[1] = b.template at(p + 8); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - rules[i].v.frameSize[0] = b.template at(p); - rules[i].v.frameSize[0] = b.template at(p + 2); + rules[ruleCount].v.frameSize[0] = b.template at(p); + rules[ruleCount].v.frameSize[0] = b.template at(p + 2); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: - rules[i].v.tag.id = b.template at(p); - rules[i].v.tag.value = b.template at(p + 4); + rules[ruleCount].v.tag.id = b.template at(p); + rules[ruleCount].v.tag.value = b.template at(p + 4); break; } p += fieldLen; + ++ruleCount; } } @@ -350,6 +347,7 @@ public: b.append(_expiration); b.append(_id); + b.append((uint16_t)_ruleCount); serializeRules(b,_rules,_ruleCount); b.append((uint8_t)_maxCustodyChainLength); @@ -387,7 +385,10 @@ public: _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; - deserializeRules(b,p,_rules,_ruleCount,ZT_MAX_CAPABILITY_RULES); + const unsigned int rc = b.template at(p); p += 2; + if (rc > ZT_MAX_CAPABILITY_RULES) + throw std::runtime_error("rule overflow"); + deserializeRules(b,p,_rules,_ruleCount,rc); _maxCustodyChainLength = (unsigned int)b[p++]; if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 53f6b88a..5c9e80f8 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -84,7 +84,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } const Packet::Verb v = verb(); - TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); + //TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" @@ -401,8 +401,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p } break; case Packet::VERB_NETWORK_CONFIG_REQUEST: { - const SharedPtr nw(RR->node->network(at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID))); - if ((nw)&&(nw->controller() == peer->address())) { + const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(nwid)); + if ((network)&&(network->controller() == peer->address())) { const unsigned int chunkLen = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN); const void *chunkData = field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,chunkLen); unsigned int chunkIndex = 0; @@ -411,7 +412,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p totalSize = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen); chunkIndex = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen + 4); } - nw->handleInboundConfigChunk(inRePacketId,chunkData,chunkLen,chunkIndex,totalSize); + TRACE("%s(%s): OK(NETWORK_CONFIG_REQUEST) chunkLen==%u chunkIndex==%u totalSize==%u",source().toString().c_str(),_remoteAddress.toString().c_str(),chunkLen,chunkIndex,totalSize); + network->handleInboundConfigChunk(inRePacketId,chunkData,chunkLen,chunkIndex,totalSize); } } break; @@ -500,33 +502,32 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - if (RR->topology->isUpstream(peer->identity())) { // only upstream peers can tell us to rendezvous, otherwise this opens a potential amplification attack vector - const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr withPeer(RR->topology->getPeer(with)); - if (withPeer) { - const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); - const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; - if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP); - - const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - TRACE("RENDEZVOUS from %s says %s might be at %s, attempting to contact",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); - if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) { - const uint64_t now = RR->node->now(); - peer->sendHELLO(_localAddress,atAddr,now,2); // send low-TTL packet to 'open' local NAT(s) - if (!peer->pushDirectPaths(_localAddress,atAddr,now,true)) - peer->sendHELLO(_localAddress,atAddr,now); - } + const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + const SharedPtr withPeer(RR->topology->getPeer(with)); + if (withPeer) { + const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); + const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; + if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { + const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); + if (!RR->topology->isUpstream(peer->identity())) { + TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + } else if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) { + const uint64_t now = RR->node->now(); + peer->sendHELLO(_localAddress,atAddr,now,2); // send low-TTL packet to 'open' local NAT(s) + if (!peer->pushDirectPaths(_localAddress,atAddr,now,true)) + peer->sendHELLO(_localAddress,atAddr,now); + TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { - TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } } else { - RR->sw->requestWhois(with); - TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),with.toString().c_str()); + TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } } else { - TRACE("ignored RENDEZVOUS from %s(%s): not a root server or a network relay",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),with.toString().c_str()); } + + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP); } catch ( ... ) { TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } diff --git a/node/Network.cpp b/node/Network.cpp index b9a2ca1d..4d588a30 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -593,13 +593,15 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize) { std::string newConfig; - if ((_inboundConfigPacketId == inRePacketId)&&(totalSize < ZT_NETWORKCONFIG_DICT_CAPACITY)&&((chunkIndex + chunkSize) < totalSize)) { + if ((_inboundConfigPacketId == inRePacketId)&&(totalSize < ZT_NETWORKCONFIG_DICT_CAPACITY)&&((chunkIndex + chunkSize) <= totalSize)) { Mutex::Lock _l(_lock); - TRACE("got %u bytes at position %u of network config request %.16llx, total expected length %u",chunkSize,chunkIndex,inRePacketId,totalSize); + _inboundConfigChunks[chunkIndex].append((const char *)data,chunkSize); + unsigned int totalWeHave = 0; for(std::map::iterator c(_inboundConfigChunks.begin());c!=_inboundConfigChunks.end();++c) totalWeHave += (unsigned int)c->second.length(); + if (totalWeHave == totalSize) { TRACE("have all chunks for network config request %.16llx, assembling...",inRePacketId); for(std::map::iterator c(_inboundConfigChunks.begin());c!=_inboundConfigChunks.end();++c) @@ -610,23 +612,29 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d _inboundConfigPacketId = 0; _inboundConfigChunks.clear(); } + } else { + return; } - if (newConfig.length() > 0) { - if (newConfig.length() < ZT_NETWORKCONFIG_DICT_CAPACITY) { - Dictionary *dict = new Dictionary(newConfig.c_str()); - try { - Identity controllerId(RR->topology->getIdentity(this->controller())); - if (controllerId) { - NetworkConfig nc; - if (nc.fromDictionary(controllerId,*dict)) - this->setConfiguration(nc,true); + if ((newConfig.length() > 0)&&(newConfig.length() < ZT_NETWORKCONFIG_DICT_CAPACITY)) { + Dictionary *dict = new Dictionary(newConfig.c_str()); + NetworkConfig *nc = new NetworkConfig(); + try { + Identity controllerId(RR->topology->getIdentity(this->controller())); + if (controllerId) { + if (nc->fromDictionary(controllerId,*dict)) { + this->setConfiguration(*nc,true); + } else { + TRACE("error parsing new config with length %u: deserialization of NetworkConfig failed (certificate error?)",(unsigned int)newConfig.length()); } - delete dict; - } catch ( ... ) { - delete dict; - throw; } + delete nc; + delete dict; + } catch ( ... ) { + TRACE("error parsing new config with length %u: unexpected exception",(unsigned int)newConfig.length()); + delete nc; + delete dict; + throw; } } } -- cgit v1.2.3 From 7d906df8057fa869ab6248b3fb215a94eaf0bffa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 10 Aug 2016 14:27:52 -0700 Subject: Better instrumentation for filter, and filter bug fixes. --- node/Network.cpp | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 4d588a30..0bd4ea55 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -37,6 +37,36 @@ namespace ZeroTier { +#ifdef ZT_TRACE +static const char *_rtn(const ZT_VirtualNetworkRuleType rt) +{ + switch(rt) { + case ZT_NETWORK_RULE_ACTION_DROP: return "ACTION_DROP"; + case ZT_NETWORK_RULE_ACTION_ACCEPT: return "ACTION_ACCEPT"; + case ZT_NETWORK_RULE_ACTION_TEE: return "ACTION_TEE"; + case ZT_NETWORK_RULE_ACTION_REDIRECT: return "ACTION_REDIRECT"; + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: return "MATCH_SOURCE_ZEROTIER_ADDRESS"; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: return "MATCH_DEST_ZEROTIER_ADDRESS"; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: return "MATCH_VLAN_PCP"; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: return "MATCH_VLAN_DEI"; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: return "MATCH_ETHERTYPE"; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: return "MATCH_MAC_SOURCE"; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: return "MATCH_MAC_DEST"; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: return "MATCH_IPV4_SOURCE"; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: return "MATCH_IPV4_DEST"; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: return "MATCH_IPV6_SOURCE"; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: return "MATCH_IPV6_DEST"; + case ZT_NETWORK_RULE_MATCH_IP_TOS: return "MATCH_IP_TOS"; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: return "MATCH_IP_PROTOCOL"; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; + default: return "BAD_RULE_TYPE"; + } +} +#endif // ZT_TRACE + // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) { @@ -66,6 +96,9 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } +//#define FILTER_TRACE TRACE +#define FILTER_TRACE(f,...) {} + // 0 == no match, -1 == match/drop, 1 == match/accept static int _doZtFilter( const RuntimeEnvironment *RR, @@ -107,6 +140,7 @@ static int _doZtFilter( if (thisSetMatches) { return -1; // match, drop packet } else { + thisRuleMatches = 1; thisSetMatches = 1; // no match, evaluate next set } break; @@ -114,6 +148,7 @@ static int _doZtFilter( if (thisSetMatches) { return 1; // match, accept packet } else { + thisRuleMatches = 1; thisSetMatches = 1; // no match, evaluate next set } break; @@ -132,6 +167,7 @@ static int _doZtFilter( if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { return -1; // match, drop packet (we redirected it) } else { + thisRuleMatches = 1; thisSetMatches = 1; // TEE does not terminate evaluation } } break; @@ -141,32 +177,41 @@ static int _doZtFilter( // thisSetMatches is the binary AND of the result of all rules in a set case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + FILTER_TRACE("FILTER[%u] %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + FILTER_TRACE("FILTER[%u] %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: + FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: // NOT SUPPORTED YET + FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: // NOT SUPPORTED YET + FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + FILTER_TRACE("FILTER[%u] %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + FILTER_TRACE("FILTER[%u] %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: + FILTER_TRACE("FILTER[%u] %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); } else { @@ -174,6 +219,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); } else { @@ -181,6 +227,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); } else { @@ -188,6 +235,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); } else { @@ -195,6 +243,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IP_TOS: + FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { @@ -205,6 +254,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); } else if (etherType == ZT_ETHERTYPE_IPV6) { @@ -236,6 +286,7 @@ static int _doZtFilter( } break; } + FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; @@ -254,11 +305,14 @@ static int _doZtFilter( } break; } + FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; } else { + FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; } } else { + FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; } break; @@ -277,15 +331,18 @@ static int _doZtFilter( } } } + FILTER_TRACE("FILTER[%u] %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { + FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); const Tag *lt = (const Tag *)0; for(unsigned int i=0;i> 7)); - //TRACE("[%u] %u result==%u set==%u",rn,(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); + FILTER_TRACE("FILTER[%u] %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); } return 0; -- cgit v1.2.3 From bd15262e5459c6003e54bcfd1d98966ff6bd1f97 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 15 Aug 2016 18:49:50 -0700 Subject: Bunch of rule JSON stuff. --- controller/SqliteNetworkController.cpp | 440 ++++++++++++++++++++++++++++++--- node/InetAddress.hpp | 13 + 2 files changed, 420 insertions(+), 33 deletions(-) (limited to 'node') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 226da657..16c02295 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -81,6 +81,271 @@ using json = nlohmann::json; namespace ZeroTier { +struct json::object _renderRule(ZT_VirtualNetworkRule &rule) +{ + char tmp[128]; + json::object r; + r["not"] = ((rule.t & 0x80) != 0); + switch((rule.t) & 0x7f) { + case ZT_NETWORK_RULE_ACTION_DROP: + r["type"] = "ACTION_DROP"; + break; + case ZT_NETWORK_RULE_ACTION_ACCEPT: + r["type"] = "ACTION_ACCEPT"; + break; + case ZT_NETWORK_RULE_ACTION_TEE: + r["type"] = "ACTION_TEE"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_ACTION_REDIRECT: + r["type"] = "ACTION_REDIRECT"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + r["type"] = "MATCH_VLAN_ID"; + r["vlanId"] = (uint64_t)rule.v.vlanId; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + r["type"] = "MATCH_VLAN_PCP"; + r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + r["type"] = "MATCH_VLAN_DEI"; + r["vlanDei"] = (uint64_t)rule.v.vlanDei; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + r["type"] = "MATCH_ETHERTYPE"; + r["etherType"] = (uint64_t)rule.v.etherType; + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + r["type"] = "MATCH_MAC_SOURCE"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + r["type"] = "MATCH_MAC_DEST"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + r["type"] = "MATCH_IPV4_SOURCE"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + r["type"] = "MATCH_IPV4_DEST"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + r["type"] = "MATCH_IPV6_SOURCE"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + r["type"] = "MATCH_IPV6_DEST"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + r["type"] = "MATCH_IP_TOS"; + r["ipTos"] = (uint64_t)rule.v.ipTos; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + r["type"] = "MATCH_IP_PROTOCOL"; + r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + r["type"] = "MATCH_CHARACTERISTICS"; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); + r["mask"] = tmp; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); + r["value"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["start"] = (uint64_t)rule.v.frameSize[0]; + r["end"] = (uint64_t)rule.v.frameSize[1]; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + r["type"] = "MATCH_TAGS_SAMENESS"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + } + return r; +} + +struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) +{ + std::string t = r["type"]; + memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); + if (r.value("not",false)) + rule.t = 0x80; + else rule.t = 0x00; + if (t == "ACTION_DROP") { + rule.t |= ZT_NETWORK_RULE_ACTION_DROP; + return true; + } else if (t == "ACTION_ACCEPT") { + rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT; + return true; + } else if (t == "ACTION_TEE") { + rule.t |= ZT_NETWORK_RULE_ACTION_TEE; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "ACTION_REDIRECT") { + rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_VLAN_ID") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; + rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_VLAN_PCP") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; + rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_VLAN_DEI") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; + rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_ETHERTYPE") { + rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; + rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_MAC_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_MAC_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_IPV4_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV4_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV6_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IPV6_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IP_TOS") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; + rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_PROTOCOL") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; + rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_IP_DEST_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_CHARACTERISTICS") { + rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; + if (r.count("mask")) { + auto mask = r["mask"]; + rule.v.characteristics[0] = (mask.is_integer() ? (uint64_t)mask : Utils::hexStrToU64(mask)); + } + if (r.count("value")) { + auto value = r["value"]; + rule.v.characteristics[1] = (value.is_integer() ? (uint64_t)value : Utils::hexStrToU64(value)); + } + return true; + } else if (t == "MATCH_FRAME_SIZE_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; + rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0")) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0")) & 0xffffULL); + return true; + } else if (t == "MATCH_TAGS_SAMENESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_AND") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_OR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_XOR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + return true; + } + return false; +} + SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath) : _node(node), _db(dbPath), @@ -1050,49 +1315,157 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } // else 404 } else { - std::vector path_copy(path); + // POST to network ID - if (!networkExists) { - if (path[1].substr(10) == "______") { - // A special POST /network/##########______ feature lets users create a network - // with an arbitrary unused network number at this controller. - nwid = 0; + json::object b; + try { + b = json::parse(body); + } catch ( ... ) { + return 403; + } - uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; - uint64_t nwidPostfix = 0; + // Magic ID ending with ______ picks a random unused network ID + if (path[1].substr(10) == "______") { + nwid = 0; + uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; + uint64_t nwidPostfix = 0; + for(unsigned long k=0;k<100000;++k) { // sanity limit on trials Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); - uint64_t nwidOriginalPostfix = nwidPostfix; - do { - uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); - if (!nwidPostfix) - tryNwid |= 1; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - - sqlite3_reset(_sGetNetworkRevision); - sqlite3_bind_text(_sGetNetworkRevision,1,nwids,16,SQLITE_STATIC); - if (sqlite3_step(_sGetNetworkRevision) != SQLITE_ROW) { - nwid = tryNwid; - break; + uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); + if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); + if (!networks.get(nwids)) { + nwid = tryNwid; + break; + } + } + if (!nwid) + return 503; + } + + json::object &network = networks[nwids]; + if (!network.is_object()) network = json::object(); + + try { + if (b.count("name")) network["name"] = b.get("name"); + if (b.count("private")) network["private"] = b.get("private"); + if (b.count("enableBroadcast")) network["enableBroadcast"] = b.get("enableBroadcast"); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.get("allowPassiveBridging"); + if (b.count("multicastLimit")) network["multicastLimit"] = b.get("multicastLimit"); + + if (b.count("v4AssignMode")) { + auto nv4m = network["v4AssignMode"]; + if (!nv4m.is_object()) nv6m = json::object(); + if (b["v4AssignMode"].is_string()) { // backward compatibility + nv4m["zt"] = (b.get("v4AssignMode") == "zt"); + } else if (b["v4AssignMode"].is_object()) { + auto v4m = b["v4AssignMode"]; + if (v4m.count("zt")) nv4m["zt"] = v4m.get("zt"); + } + if (!nv4m.count("zt")) nv4m["zt"] = false; + } + + if (b.count("v6AssignMode")) { + auto nv6m = network["v6AssignMode"]; + if (!nv6m.is_object()) nv6m = json::object(); + if (b["v6AssignMode"].is_string()) { // backward compatibility + std::vector v6m(Utils::split(b.get("v6AssignMode").c_str(),",","","")); + std::sort(v6m.begin(),v6m.end()); + v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); + for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { + if (*i == "rfc4193") + nv6m["rfc4193"] = true; + else if (*i == "zt") + nv6m["zt"] = true; + else if (*i == "6plane") + nv6m["6plane"] = true; } + } else if (b["v6AssignMode"].is_object()) { + auto v6m = b["v6AssignMode"]; + if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.get("rfc4193"); + if (v6m.count("zt")) nv6m["rfc4193"] = v6m.get("zt"); + if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.get("6plane"); + } + if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; + if (!nv6m.count("zt")) nv6m["zt"] = false; + if (!nv6m.count("6plane")) nv6m["6plane"] = false; + } - ++nwidPostfix; - } while (nwidPostfix != nwidOriginalPostfix); + if (b.count("routes")) { + auto rts = b["routes"]; + if (rts.is_array()) { + for(unsigned long i=0;i("target")); + InetAddress v(rt.get("via")); + if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.ss_family == v.ss_family) && (t.netmaskBitsValid()) ) { + auto nrts = network["routes"]; + if (!nrts.is_array()) nrts = json::array(); + json::object tmp; + tmp["target"] = target.toString(); + tmp["via"] = target.toIpString(); + nrts.push_back(tmp); + } + } + } + } + } - // 503 means we have no more free IDs for this prefix. You shouldn't host anywhere - // near 16 million networks on the same controller, so shouldn't happen. - if (!nwid) - return 503; + if (b.count("ipAssignmentPools")) { + auto ipp = b["ipAssignmentPools"]; + if (ipp.is_array()) { + for(unsigned long i=0;i("ipRangeStart")); + InetAddress t(ip.get("ipRangeEnd")); + if ( ((f.ss_family == AF_INET)||(f.ss_family == AF_INET6)) && (f.ss_family == t.ss_family) ) { + auto nipp = network["ipAssignmentPools"]; + if (!nipp.is_array()) nipp = json::array(); + json::object tmp; + tmp["ipRangeStart"] = f.toIpString(); + tmp["ipRangeEnd"] = t.toIpString(); + nipp.push_back(tmp); + } + } + } + } } - sqlite3_reset(_sCreateNetwork); - sqlite3_bind_text(_sCreateNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sCreateNetwork,2,"",0,SQLITE_STATIC); - sqlite3_bind_int64(_sCreateNetwork,3,(long long)OSUtils::now()); - if (sqlite3_step(_sCreateNetwork) != SQLITE_DONE) - return 500; - path_copy[1].assign(nwids); + if (b.count("rules")) { + auto rules = b["rules"]; + if (rules.is_array()) { + for(unsigned long i=0;i("revision") + 1ULL; + + return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); + + /* json_value *j = json_parse(body.c_str(),body.length()); if (j) { if (j->type == json_object) { @@ -1409,6 +1782,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); } + */ } // else 404 diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index b97e2ca6..2cd4dbd3 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -300,6 +300,19 @@ struct InetAddress : public sockaddr_storage */ inline unsigned int netmaskBits() const throw() { return port(); } + /** + * @return True if netmask bits is valid for the address type + */ + inline bool netmaskBitsValid() const + { + const unsigned int n = port(); + switch(ss_family) { + case AF_INET: return (n <= 32); + case AF_INET6: return (n <= 128); + } + return false; + } + /** * Alias for port() * -- cgit v1.2.3 From b08ca49580c63abe79a650adbb0d14ca87a1cd24 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 16 Aug 2016 14:05:17 -0700 Subject: More controller work -- it builds! --- controller/SqliteNetworkController.cpp | 729 +-- controller/SqliteNetworkController.hpp | 77 +- ext/json/LICENSE.MIT | 22 + ext/json/README.md | 511 ++ ext/json/json.hpp | 10435 +++++++++++++++++++++++++++++++ ext/offbase/README.md | 59 - ext/offbase/json/LICENSE.MIT | 22 - ext/offbase/json/README.md | 511 -- ext/offbase/json/json.hpp | 10435 ------------------------------- ext/offbase/offbase.hpp | 393 -- make-mac.mk | 4 +- node/InetAddress.cpp | 2 +- osdep/OSUtils.cpp | 80 + osdep/OSUtils.hpp | 18 +- 14 files changed, 11280 insertions(+), 12018 deletions(-) create mode 100644 ext/json/LICENSE.MIT create mode 100644 ext/json/README.md create mode 100644 ext/json/json.hpp delete mode 100644 ext/offbase/README.md delete mode 100644 ext/offbase/json/LICENSE.MIT delete mode 100644 ext/offbase/json/README.md delete mode 100644 ext/offbase/json/json.hpp delete mode 100644 ext/offbase/offbase.hpp (limited to 'node') diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp index 16c02295..94d2e2e6 100644 --- a/controller/SqliteNetworkController.cpp +++ b/controller/SqliteNetworkController.cpp @@ -53,8 +53,6 @@ #include "../node/MAC.hpp" #include "../node/Address.hpp" -#include "../osdep/OSUtils.hpp" - // offbase includes and builds upon nlohmann::json using json = nlohmann::json; @@ -73,18 +71,15 @@ using json = nlohmann::json; // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 -// Delay between backups in milliseconds -//#define ZT_NETCONF_BACKUP_PERIOD 300000 - // Nodes are considered active if they've queried in less than this long #define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) namespace ZeroTier { -struct json::object _renderRule(ZT_VirtualNetworkRule &rule) +static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; - json::object r; + json r = json::object(); r["not"] = ((rule.t & 0x80) != 0); switch((rule.t) & 0x7f) { case ZT_NETWORK_RULE_ACTION_DROP: @@ -205,8 +200,10 @@ struct json::object _renderRule(ZT_VirtualNetworkRule &rule) return r; } -struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) +static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) { + if (r.is_object()) + return false; std::string t = r["type"]; memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); if (r.value("not",false)) @@ -220,19 +217,19 @@ struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0")) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_VLAN_ID") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; @@ -309,48 +306,58 @@ struct bool _parseRule(const json::object &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; if (r.count("mask")) { - auto mask = r["mask"]; - rule.v.characteristics[0] = (mask.is_integer() ? (uint64_t)mask : Utils::hexStrToU64(mask)); + auto v = r["mask"]; + if (v.is_number()) { + rule.v.characteristics[0] = v; + } else { + std::string tmp = v; + rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); + } } if (r.count("value")) { - auto value = r["value"]; - rule.v.characteristics[1] = (value.is_integer() ? (uint64_t)value : Utils::hexStrToU64(value)); + auto v = r["value"]; + if (v.is_number()) { + rule.v.characteristics[1] = v; + } else { + std::string tmp = v; + rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); + } } return true; } else if (t == "MATCH_FRAME_SIZE_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0")) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0")) & 0xffffULL); + rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); return true; } else if (t == "MATCH_TAGS_SAMENESS") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0")) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0")) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); return true; } return false; } -SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath) : +SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath) : _node(node), - _db(dbPath), - _dbCommitThreadRun(true) + _path(dbPath) { + OSUtils::mkdir(dbPath); /* if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) throw std::runtime_error("SqliteNetworkController cannot open database file"); @@ -592,16 +599,10 @@ SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,c _backupThread = Thread::start(this); */ - - _dbCommitThread = Thread::start(this); } SqliteNetworkController::~SqliteNetworkController() { - _lock.lock(); - _dbCommitThreadRun = false; - _lock.unlock(); - Thread::join(_dbCommitThread); } NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) @@ -1068,7 +1069,6 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpGET( std::string &responseBody, std::string &responseContentType) { - Mutex::Lock _l(_lock); return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); } @@ -1082,41 +1082,21 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( { if (path.empty()) return 404; - Mutex::Lock _l(_lock); if (path[0] == "network") { - json &networks = _db["network"]; - if (!networks.is_object()) networks = json::object(); if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - /* - int64_t revision = 0; - sqlite3_reset(_sGetNetworkRevision); - sqlite3_bind_text(_sGetNetworkRevision,1,nwids,16,SQLITE_STATIC); - bool networkExists = false; - if (sqlite3_step(_sGetNetworkRevision) == SQLITE_ROW) { - networkExists = true; - revision = sqlite3_column_int64(_sGetNetworkRevision,0); - } - */ - if (path.size() >= 3) { - auto network = networks.get(nwids); - if (!network) return 404; + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - json &members = (*network)["member"]; - if (!members.is_object()) members = json::object(); - uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - json &member = members[addrs]; - if (!member.is_object()) member = json::object(); /* int64_t addToNetworkRevision = 0; @@ -1256,6 +1236,9 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType); } else if ((path.size() == 3)&&(path[2] == "test")) { + + Mutex::Lock _l(_circuitTests_m); + ZT_CircuitTest *test = (ZT_CircuitTest *)malloc(sizeof(ZT_CircuitTest)); memset(test,0,sizeof(ZT_CircuitTest)); @@ -1263,6 +1246,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( test->credentialNetworkId = nwid; test->ptr = (void *)this; + // TODO TODO /* json_value *j = json_parse(body.c_str(),body.length()); if (j) { @@ -1312,12 +1296,13 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( responseBody = json; responseContentType = "application/json"; return 200; + } // else 404 } else { // POST to network ID - json::object b; + json b; try { b = json::parse(body); } catch ( ... ) { @@ -1334,7 +1319,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (!networks.get(nwids)) { + if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { nwid = tryNwid; break; } @@ -1343,24 +1328,23 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( return 503; } - json::object &network = networks[nwids]; - if (!network.is_object()) network = json::object(); + json network(_readJson(_networkJP(nwid,true))); try { - if (b.count("name")) network["name"] = b.get("name"); - if (b.count("private")) network["private"] = b.get("private"); - if (b.count("enableBroadcast")) network["enableBroadcast"] = b.get("enableBroadcast"); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.get("allowPassiveBridging"); - if (b.count("multicastLimit")) network["multicastLimit"] = b.get("multicastLimit"); + if (b.count("name")) network["name"] = b.value("name",""); + if (b.count("private")) network["private"] = b.value("private",true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); + if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); if (b.count("v4AssignMode")) { auto nv4m = network["v4AssignMode"]; - if (!nv4m.is_object()) nv6m = json::object(); + if (!nv4m.is_object()) nv4m = json::object(); if (b["v4AssignMode"].is_string()) { // backward compatibility - nv4m["zt"] = (b.get("v4AssignMode") == "zt"); + nv4m["zt"] = (b.value("v4AssignMode","") == "zt"); } else if (b["v4AssignMode"].is_object()) { auto v4m = b["v4AssignMode"]; - if (v4m.count("zt")) nv4m["zt"] = v4m.get("zt"); + if (v4m.count("zt")) nv4m["zt"] = v4m.value("zt",false); } if (!nv4m.count("zt")) nv4m["zt"] = false; } @@ -1369,7 +1353,7 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( auto nv6m = network["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (b["v6AssignMode"].is_string()) { // backward compatibility - std::vector v6m(Utils::split(b.get("v6AssignMode").c_str(),",","","")); + std::vector v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); std::sort(v6m.begin(),v6m.end()); v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { @@ -1382,9 +1366,9 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } } else if (b["v6AssignMode"].is_object()) { auto v6m = b["v6AssignMode"]; - if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.get("rfc4193"); - if (v6m.count("zt")) nv6m["rfc4193"] = v6m.get("zt"); - if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.get("6plane"); + if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); + if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); + if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); } if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; if (!nv6m.count("zt")) nv6m["zt"] = false; @@ -1397,14 +1381,14 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i("target")); - InetAddress v(rt.get("via")); + InetAddress t(rt.value("target","")); + InetAddress v(rt.value("via","")); if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.ss_family == v.ss_family) && (t.netmaskBitsValid()) ) { auto nrts = network["routes"]; if (!nrts.is_array()) nrts = json::array(); - json::object tmp; - tmp["target"] = target.toString(); - tmp["via"] = target.toIpString(); + json tmp; + tmp["target"] = t.toString(); + tmp["via"] = v.toIpString(); nrts.push_back(tmp); } } @@ -1418,12 +1402,12 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i("ipRangeStart")); - InetAddress t(ip.get("ipRangeEnd")); + InetAddress f(ip.value("ipRangeStart","")); + InetAddress t(ip.value("ipRangeEnd","")); if ( ((f.ss_family == AF_INET)||(f.ss_family == AF_INET6)) && (f.ss_family == t.ss_family) ) { auto nipp = network["ipAssignmentPools"]; if (!nipp.is_array()) nipp = json::array(); - json::object tmp; + json tmp; tmp["ipRangeStart"] = f.toIpString(); tmp["ipRangeEnd"] = t.toIpString(); nipp.push_back(tmp); @@ -1461,328 +1445,14 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( } network["lastModified"] = OSUtils::now(); - network["revision"] = network.get("revision") + 1ULL; - - return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); - - /* - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - sqlite3_stmt *stmt = (sqlite3_stmt *)0; - - if (!strcmp(j->u.object.values[k].name,"name")) { - if ((j->u.object.values[k].value->type == json_string)&&(j->u.object.values[k].value->u.string.ptr[0])) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"name\" = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_text(stmt,1,j->u.object.values[k].value->u.string.ptr,-1,SQLITE_STATIC); - } - } else if (!strcmp(j->u.object.values[k].name,"private")) { - if (j->u.object.values[k].value->type == json_boolean) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"private\" = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - } - } else if (!strcmp(j->u.object.values[k].name,"enableBroadcast")) { - if (j->u.object.values[k].value->type == json_boolean) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET enableBroadcast = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - } - } else if (!strcmp(j->u.object.values[k].name,"allowPassiveBridging")) { - if (j->u.object.values[k].value->type == json_boolean) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET allowPassiveBridging = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(j->u.object.values[k].value->u.boolean == 0) ? 0 : 1); - } - } else if (!strcmp(j->u.object.values[k].name,"v4AssignMode")) { - if ((j->u.object.values[k].value->type == json_string)&&(!strcmp(j->u.object.values[k].value->u.string.ptr,"zt"))) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"flags\" = (\"flags\" | ?) WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(int)ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN); - } else { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"flags\" = (\"flags\" & ?) WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(int)(ZT_DB_NETWORK_FLAG_ZT_MANAGED_V4_AUTO_ASSIGN ^ 0xfffffff)); - } - } else if (!strcmp(j->u.object.values[k].name,"v6AssignMode")) { - int fl = 0; - if (j->u.object.values[k].value->type == json_string) { - char *saveptr = (char *)0; - for(char *f=Utils::stok(j->u.object.values[k].value->u.string.ptr,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) { - if (!strcmp(f,"rfc4193")) - fl |= ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_RFC4193; - else if (!strcmp(f,"6plane")) - fl |= ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_6PLANE; - else if (!strcmp(f,"zt")) - fl |= ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_AUTO_ASSIGN; - } - } - if (sqlite3_prepare_v2(_db,"UPDATE Network SET \"flags\" = ((\"flags\" & " ZT_DB_NETWORK_FLAG_ZT_MANAGED_V6_MASK_S ") | ?) WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,fl); - } else if (!strcmp(j->u.object.values[k].name,"multicastLimit")) { - if (j->u.object.values[k].value->type == json_integer) { - if (sqlite3_prepare_v2(_db,"UPDATE Network SET multicastLimit = ? WHERE id = ?",-1,&stmt,(const char **)0) == SQLITE_OK) - sqlite3_bind_int(stmt,1,(int)j->u.object.values[k].value->u.integer); - } - } else if (!strcmp(j->u.object.values[k].name,"relays")) { - if (j->u.object.values[k].value->type == json_array) { - std::map nodeIdToPhyAddress; - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *relay = j->u.object.values[k].value->u.array.values[kk]; - const char *address = (const char *)0; - const char *phyAddress = (const char *)0; - if ((relay)&&(relay->type == json_object)) { - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(relay->u.object.values[rk].name,"address"))&&(relay->u.object.values[rk].value->type == json_string)) - address = relay->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(relay->u.object.values[rk].name,"phyAddress"))&&(relay->u.object.values[rk].value->type == json_string)) - phyAddress = relay->u.object.values[rk].value->u.string.ptr; - } - } - if ((address)&&(phyAddress)) - nodeIdToPhyAddress[Address(address)] = InetAddress(phyAddress); - } - - sqlite3_reset(_sDeleteRelaysForNetwork); - sqlite3_bind_text(_sDeleteRelaysForNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteRelaysForNetwork); - - for(std::map::iterator rl(nodeIdToPhyAddress.begin());rl!=nodeIdToPhyAddress.end();++rl) { - sqlite3_reset(_sCreateRelay); - sqlite3_bind_text(_sCreateRelay,1,nwids,16,SQLITE_STATIC); - std::string a(rl->first.toString()),b(rl->second.toString()); // don't destroy strings until sqlite3_step() - sqlite3_bind_text(_sCreateRelay,2,a.c_str(),-1,SQLITE_STATIC); - sqlite3_bind_text(_sCreateRelay,3,b.c_str(),-1,SQLITE_STATIC); - sqlite3_step(_sCreateRelay); - } - } - } else if (!strcmp(j->u.object.values[k].name,"routes")) { - sqlite3_reset(_sDeleteRoutes); - sqlite3_bind_text(_sDeleteRoutes,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteRoutes); - if (j->u.object.values[k].value->type == json_array) { - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *r = j->u.object.values[k].value->u.array.values[kk]; - if ((r)&&(r->type == json_object)) { - InetAddress r_target,r_via; - int r_flags = 0; - int r_metric = 0; - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(r->u.object.values[rk].name,"target"))&&(r->u.object.values[rk].value->type == json_string)) - r_target = InetAddress(std::string(r->u.object.values[rk].value->u.string.ptr)); - else if ((!strcmp(r->u.object.values[rk].name,"via"))&&(r->u.object.values[rk].value->type == json_string)) - r_via = InetAddress(std::string(r->u.object.values[rk].value->u.string.ptr),0); - else if ((!strcmp(r->u.object.values[rk].name,"flags"))&&(r->u.object.values[rk].value->type == json_integer)) - r_flags = (int)(r->u.object.values[rk].value->u.integer & 0xffff); - else if ((!strcmp(r->u.object.values[rk].name,"metric"))&&(r->u.object.values[rk].value->type == json_integer)) - r_metric = (int)(r->u.object.values[rk].value->u.integer & 0xffff); - } - if ((r_target)&&((!r_via)||(r_via.ss_family == r_target.ss_family))) { - int r_ipVersion = 0; - char r_targetBlob[16]; - char r_viaBlob[16]; - _ipToBlob(r_target,r_targetBlob,r_ipVersion); - if (r_ipVersion) { - int r_targetNetmaskBits = r_target.netmaskBits(); - if ((r_ipVersion == 4)&&(r_targetNetmaskBits > 32)) r_targetNetmaskBits = 32; - else if ((r_ipVersion == 6)&&(r_targetNetmaskBits > 128)) r_targetNetmaskBits = 128; - sqlite3_reset(_sCreateRoute); - sqlite3_bind_text(_sCreateRoute,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_blob(_sCreateRoute,2,(const void *)r_targetBlob,16,SQLITE_STATIC); - if (r_via) { - _ipToBlob(r_via,r_viaBlob,r_ipVersion); - sqlite3_bind_blob(_sCreateRoute,3,(const void *)r_viaBlob,16,SQLITE_STATIC); - } else { - sqlite3_bind_null(_sCreateRoute,3); - } - sqlite3_bind_int(_sCreateRoute,4,r_targetNetmaskBits); - sqlite3_bind_int(_sCreateRoute,5,r_ipVersion); - sqlite3_bind_int(_sCreateRoute,6,r_flags); - sqlite3_bind_int(_sCreateRoute,7,r_metric); - sqlite3_step(_sCreateRoute); - } - } - } - } - } - } else if (!strcmp(j->u.object.values[k].name,"ipAssignmentPools")) { - if (j->u.object.values[k].value->type == json_array) { - std::vector< std::pair > pools; - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *pool = j->u.object.values[k].value->u.array.values[kk]; - const char *iprs = (const char *)0; - const char *ipre = (const char *)0; - if ((pool)&&(pool->type == json_object)) { - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(pool->u.object.values[rk].name,"ipRangeStart"))&&(pool->u.object.values[rk].value->type == json_string)) - iprs = pool->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(pool->u.object.values[rk].name,"ipRangeEnd"))&&(pool->u.object.values[rk].value->type == json_string)) - ipre = pool->u.object.values[rk].value->u.string.ptr; - } - } - if ((iprs)&&(ipre)) { - InetAddress iprs2(iprs); - InetAddress ipre2(ipre); - if (iprs2.ss_family == ipre2.ss_family) { - iprs2.setPort(0); - ipre2.setPort(0); - pools.push_back(std::pair(iprs2,ipre2)); - } - } - } - std::sort(pools.begin(),pools.end()); - pools.erase(std::unique(pools.begin(),pools.end()),pools.end()); - - sqlite3_reset(_sDeleteIpAssignmentPoolsForNetwork); - sqlite3_bind_text(_sDeleteIpAssignmentPoolsForNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteIpAssignmentPoolsForNetwork); - - for(std::vector< std::pair >::const_iterator p(pools.begin());p!=pools.end();++p) { - char ipBlob1[16],ipBlob2[16]; - sqlite3_reset(_sCreateIpAssignmentPool); - sqlite3_bind_text(_sCreateIpAssignmentPool,1,nwids,16,SQLITE_STATIC); - if (p->first.ss_family == AF_INET) { - memset(ipBlob1,0,12); - memcpy(ipBlob1 + 12,p->first.rawIpData(),4); - memset(ipBlob2,0,12); - memcpy(ipBlob2 + 12,p->second.rawIpData(),4); - sqlite3_bind_blob(_sCreateIpAssignmentPool,2,(const void *)ipBlob1,16,SQLITE_STATIC); - sqlite3_bind_blob(_sCreateIpAssignmentPool,3,(const void *)ipBlob2,16,SQLITE_STATIC); - sqlite3_bind_int(_sCreateIpAssignmentPool,4,4); - } else if (p->first.ss_family == AF_INET6) { - sqlite3_bind_blob(_sCreateIpAssignmentPool,2,p->first.rawIpData(),16,SQLITE_STATIC); - sqlite3_bind_blob(_sCreateIpAssignmentPool,3,p->second.rawIpData(),16,SQLITE_STATIC); - sqlite3_bind_int(_sCreateIpAssignmentPool,4,6); - } else continue; - sqlite3_step(_sCreateIpAssignmentPool); - } - } - } else if (!strcmp(j->u.object.values[k].name,"rules")) { - if (j->u.object.values[k].value->type == json_array) { - sqlite3_reset(_sDeleteRulesForNetwork); - sqlite3_bind_text(_sDeleteRulesForNetwork,1,nwids,16,SQLITE_STATIC); - sqlite3_step(_sDeleteRulesForNetwork); - - for(unsigned int kk=0;kku.object.values[k].value->u.array.length;++kk) { - json_value *rj = j->u.object.values[k].value->u.array.values[kk]; - if ((rj)&&(rj->type == json_object)) { - struct { // NULL pointers indicate missing or NULL -- wildcards - const json_int_t *ruleNo; - const char *nodeId; - const char *sourcePort; - const char *destPort; - const json_int_t *vlanId; - const json_int_t *vlanPcp; - const json_int_t *etherType; - const char *macSource; - const char *macDest; - const char *ipSource; - const char *ipDest; - const json_int_t *ipTos; - const json_int_t *ipProtocol; - const json_int_t *ipSourcePort; - const json_int_t *ipDestPort; - const json_int_t *flags; - const json_int_t *invFlags; - const char *action; - } rule; - memset(&rule,0,sizeof(rule)); - - for(unsigned int rk=0;rku.object.length;++rk) { - if ((!strcmp(rj->u.object.values[rk].name,"ruleNo"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ruleNo = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"nodeId"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.nodeId = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"sourcePort"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.sourcePort = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"destPort"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.destPort = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"vlanId"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.vlanId = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"vlanPcp"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.vlanPcp = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"etherType"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.etherType = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"macSource"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.macSource = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"macDest"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.macDest = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"ipSource"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.ipSource = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"ipDest"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.ipDest = rj->u.object.values[rk].value->u.string.ptr; - else if ((!strcmp(rj->u.object.values[rk].name,"ipTos"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipTos = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"ipProtocol"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipProtocol = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"ipSourcePort"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipSourcePort = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"ipDestPort"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.ipDestPort = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"flags"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.flags = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"invFlags"))&&(rj->u.object.values[rk].value->type == json_integer)) - rule.invFlags = &(rj->u.object.values[rk].value->u.integer); - else if ((!strcmp(rj->u.object.values[rk].name,"action"))&&(rj->u.object.values[rk].value->type == json_string)) - rule.action = rj->u.object.values[rk].value->u.string.ptr; - } - - if ((rule.ruleNo)&&(rule.action)&&(rule.action[0])) { - char mactmp1[16],mactmp2[16]; - sqlite3_reset(_sCreateRule); - sqlite3_bind_text(_sCreateRule,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_int64(_sCreateRule,2,*rule.ruleNo); - - // Optional values: null by default - for(int i=3;i<=18;++i) - sqlite3_bind_null(_sCreateRule,i); - if ((rule.nodeId)&&(strlen(rule.nodeId) == 10)) sqlite3_bind_text(_sCreateRule,3,rule.nodeId,10,SQLITE_STATIC); - if ((rule.sourcePort)&&(strlen(rule.sourcePort) == 10)) sqlite3_bind_text(_sCreateRule,4,rule.sourcePort,10,SQLITE_STATIC); - if ((rule.destPort)&&(strlen(rule.destPort) == 10)) sqlite3_bind_text(_sCreateRule,5,rule.destPort,10,SQLITE_STATIC); - if (rule.vlanId) sqlite3_bind_int(_sCreateRule,6,(int)*rule.vlanId); - if (rule.vlanPcp) sqlite3_bind_int(_sCreateRule,7,(int)*rule.vlanPcp); - if (rule.etherType) sqlite3_bind_int(_sCreateRule,8,(int)*rule.etherType & (int)0xffff); - if (rule.macSource) { - MAC m(rule.macSource); - Utils::snprintf(mactmp1,sizeof(mactmp1),"%.12llx",(unsigned long long)m.toInt()); - sqlite3_bind_text(_sCreateRule,9,mactmp1,-1,SQLITE_STATIC); - } - if (rule.macDest) { - MAC m(rule.macDest); - Utils::snprintf(mactmp2,sizeof(mactmp2),"%.12llx",(unsigned long long)m.toInt()); - sqlite3_bind_text(_sCreateRule,10,mactmp2,-1,SQLITE_STATIC); - } - if (rule.ipSource) sqlite3_bind_text(_sCreateRule,11,rule.ipSource,-1,SQLITE_STATIC); - if (rule.ipDest) sqlite3_bind_text(_sCreateRule,12,rule.ipDest,-1,SQLITE_STATIC); - if (rule.ipTos) sqlite3_bind_int(_sCreateRule,13,(int)*rule.ipTos); - if (rule.ipProtocol) sqlite3_bind_int(_sCreateRule,14,(int)*rule.ipProtocol); - if (rule.ipSourcePort) sqlite3_bind_int(_sCreateRule,15,(int)*rule.ipSourcePort & (int)0xffff); - if (rule.ipDestPort) sqlite3_bind_int(_sCreateRule,16,(int)*rule.ipDestPort & (int)0xffff); - if (rule.flags) sqlite3_bind_int64(_sCreateRule,17,(int64_t)*rule.flags); - if (rule.invFlags) sqlite3_bind_int64(_sCreateRule,18,(int64_t)*rule.invFlags); - - sqlite3_bind_text(_sCreateRule,19,rule.action,-1,SQLITE_STATIC); - sqlite3_step(_sCreateRule); - } - } - } - } - } + network["revision"] = network.value("revision",0ULL) + 1ULL; - if (stmt) { - sqlite3_bind_text(stmt,2,nwids,16,SQLITE_STATIC); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - } - } - } - json_value_free(j); - } - - sqlite3_reset(_sSetNetworkRevision); - sqlite3_bind_int64(_sSetNetworkRevision,1,revision += 1); - sqlite3_bind_text(_sSetNetworkRevision,2,nwids,16,SQLITE_STATIC); - sqlite3_step(_sSetNetworkRevision); + _writeJson(_networkJP(nwid,true),network); - return _doCPGet(path_copy,urlArgs,headers,body,responseBody,responseContentType); - } - */ + responseBody = network.dump(2); + responseContentType = "application/json"; + return 200; + } // else 404 } // else 404 @@ -1801,38 +1471,32 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( { if (path.empty()) return 404; - Mutex::Lock _l(_lock); if (path[0] == "network") { - auto networks = _db.get("network"); - if (!networks) return 404; - if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - auto network = _db.get(nwids); - if (!network) return 404; + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - auto members = network->get("member"); - if (!members) return 404; - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - auto member = members->get(addrs); - if (!member) return 404; - members->erase(addrs); - responseBody = member->dump(2); + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); + + responseBody = member.dump(2); responseContentType = "application/json"; return 200; } } else { - networks->erase(nwids); - responseBody = network->dump(2); + OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); + responseBody = network.dump(2); responseContentType = "application/json"; return 200; } @@ -1843,27 +1507,6 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpDELETE( return 404; } -void SqliteNetworkController::threadMain() - throw() -{ - uint64_t lastCommit = OSUtils::now(); - while(_dbCommitThreadRun) { - Thread::sleep(200); - if ((OSUtils::now() - lastCommit) > 2000) { - lastCommit = OSUtils::now(); - try { - std::vector errors; - Mutex::Lock _l(_lock); - if (!_db.commit(&errors)) { - // TODO: handle anything really bad - } - } catch ( ... ) { - // TODO: handle anything really bad - } - } - } -} - unsigned int SqliteNetworkController::_doCPGet( const std::vector &path, const std::map &urlArgs, @@ -1874,137 +1517,79 @@ unsigned int SqliteNetworkController::_doCPGet( { // Assumes _lock is locked if ((path.size() > 0)&&(path[0] == "network")) { - auto networks = _db.get("network"); - if (!networks) return 404; if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - auto network = _db.get(nwids); - if (!network) return 404; + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; if (path.size() >= 3) { if (path[2] == "member") { - auto members = network->get("member"); - if (!members) return 404; if (path.size() >= 4) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - auto member = members->get(addrs); - if (!member) return 404; - nlohmann::json o(member); + json o(member); o["nwid"] = nwids; o["address"] = addrs; - o["controllerInstanceId"] = _instanceId; o["clock"] = OSUtils::now(); responseBody = o.dump(2); responseContentType = "application/json"; - return 200; - /* - sqlite3_reset(_sGetMember2); - sqlite3_bind_text(_sGetMember2,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetMember2,2,addrs,10,SQLITE_STATIC); - if (sqlite3_step(_sGetMember2) == SQLITE_ROW) { - const char *memberIdStr = (const char *)sqlite3_column_text(_sGetMember2,3); - - Utils::snprintf(json,sizeof(json), - "{\n" - "\t\"nwid\": \"%s\",\n" - "\t\"address\": \"%s\",\n" - "\t\"controllerInstanceId\": \"%s\",\n" - "\t\"authorized\": %s,\n" - "\t\"activeBridge\": %s,\n" - "\t\"memberRevision\": %llu,\n" - "\t\"clock\": %llu,\n" - "\t\"identity\": \"%s\",\n" - "\t\"ipAssignments\": [", - nwids, - addrs, - _instanceId.c_str(), - (sqlite3_column_int(_sGetMember2,0) > 0) ? "true" : "false", - (sqlite3_column_int(_sGetMember2,1) > 0) ? "true" : "false", - (unsigned long long)sqlite3_column_int64(_sGetMember2,2), - (unsigned long long)OSUtils::now(), - _jsonEscape(memberIdStr).c_str()); - responseBody = json; - - sqlite3_reset(_sGetIpAssignmentsForNode); - sqlite3_bind_text(_sGetIpAssignmentsForNode,1,nwids,16,SQLITE_STATIC); - sqlite3_bind_text(_sGetIpAssignmentsForNode,2,addrs,10,SQLITE_STATIC); - bool firstIp = true; - while (sqlite3_step(_sGetIpAssignmentsForNode) == SQLITE_ROW) { - int ipversion = sqlite3_column_int(_sGetIpAssignmentsForNode,2); - char ipBlob[16]; - memcpy(ipBlob,(const void *)sqlite3_column_blob(_sGetIpAssignmentsForNode,0),16); - InetAddress ip( - (const void *)(ipversion == 6 ? ipBlob : &ipBlob[12]), - (ipversion == 6 ? 16 : 4), - (unsigned int)sqlite3_column_int(_sGetIpAssignmentsForNode,1) - ); - responseBody.append(firstIp ? "\"" : ",\""); - responseBody.append(_jsonEscape(ip.toIpString())); - responseBody.push_back('"'); - firstIp = false; - } - - responseBody.append("],\n\t\"recentLog\": ["); + return 200; + } else { - const void *histb = sqlite3_column_blob(_sGetMember2,6); - if (histb) { - MemberRecentHistory rh; - rh.fromBlob((const char *)histb,sqlite3_column_bytes(_sGetMember2,6)); - for(MemberRecentHistory::const_iterator i(rh.begin());i!=rh.end();++i) { - if (i != rh.begin()) - responseBody.push_back(','); + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); + responseBody.append("\":"); + const std::string rc = member.value("memberRevision","0"); + responseBody.append(rc); } } - - responseBody.append("]\n}\n"); - - responseContentType = "application/json"; - return 200; - } // else 404 - */ - - } else { - - responseBody.push_back('{'); - for(auto i(members->begin());i!=members->end();++i) { - responseBody.append((i == members->begin()) ? "\"" : ",\""); - responseBody.append(i->key()); - responseBody.append("\":\""); - const std::string rc = i->value().value("memberRevision","0"); - responseBody.append(rc); - responseBody.append('"'); } responseBody.push_back('}'); responseContentType = "application/json"; - return 200; + return 200; } } else if ((path[2] == "active")&&(path.size() == 3)) { - responseBody.push_back('{'); - bool firstMember = true; + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; - for(auto i(members->begin());i!=members->end();++i) { - auto recentLog = i->value()->get("recentLog"); - if ((recentLog)&&(recentLog.size() > 0)) { - auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { - responseBody.append((firstMember) ? "\"" : ",\""); - firstMember = false; - responseBody.append(i->key()); - responseBody.append("\":"); - responseBody.append(mostRecentLog.dump()); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + auto recentLog = member["recentLog"]; + if ((recentLog.is_array())&&(recentLog.size() > 0)) { + auto mostRecentLog = recentLog[0]; + if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + responseBody.append(mostRecentLog.dump()); + } + } } } } @@ -2014,6 +1599,7 @@ unsigned int SqliteNetworkController::_doCPGet( } else if ((path[2] == "test")&&(path.size() >= 4)) { + Mutex::Lock _l(_circuitTests_m); std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); if ((cte != _circuitTests.end())&&(cte->second.test)) { @@ -2032,7 +1618,6 @@ unsigned int SqliteNetworkController::_doCPGet( nlohmann::json o(network); o["nwid"] = nwids; - o["controllerInstanceId"] = _instanceId; o["clock"] = OSUtils::now(); responseBody = o.dump(2); responseContentType = "application/json"; @@ -2042,10 +1627,13 @@ unsigned int SqliteNetworkController::_doCPGet( } else if (path.size() == 1) { responseBody = "["; - for(auto i(networks->begin());i!=networks.end();++i) { - responseBody.append((i == networks->begin()) ? "\"" : ",\""); - responseBody.append(i->key()); - responseBody.append("\""); + std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); + for(auto i(networks.begin());i!=networks.end();++i) { + if (i->length() == 16) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); + } } responseBody.push_back(']'); responseContentType = "application/json"; @@ -2053,16 +1641,11 @@ unsigned int SqliteNetworkController::_doCPGet( } // else 404 - } else if ((path.size() > 0)&&(path[0] == "_dump")) { - - responseBody = _db.dump(2); - responseContentType = "application/json"; - return 200; - } else { - Utils::snprintf(json,sizeof(json),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu,\n\t\"instanceId\": \"%s\"\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now(),_instanceId.c_str()); - responseBody = json; + char tmp[4096]; + Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + responseBody = tmp; responseContentType = "application/json"; return 200; @@ -2081,7 +1664,7 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest if (!report) return; - Mutex::Lock _l(self->_lock); + Mutex::Lock _l(self->_circuitTests_m); std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? @@ -2092,25 +1675,25 @@ void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest Utils::snprintf(tmp,sizeof(tmp), "%s{\n" - "\t\"timestamp\": %llu,"ZT_EOL_S - "\t\"testId\": \"%.16llx\","ZT_EOL_S - "\t\"upstream\": \"%.10llx\","ZT_EOL_S - "\t\"current\": \"%.10llx\","ZT_EOL_S - "\t\"receivedTimestamp\": %llu,"ZT_EOL_S - "\t\"remoteTimestamp\": %llu,"ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\","ZT_EOL_S - "\t\"flags\": %llu,"ZT_EOL_S - "\t\"sourcePacketHopCount\": %u,"ZT_EOL_S - "\t\"errorCode\": %u,"ZT_EOL_S - "\t\"vendor\": %d,"ZT_EOL_S - "\t\"protocolVersion\": %u,"ZT_EOL_S - "\t\"majorVersion\": %u,"ZT_EOL_S - "\t\"minorVersion\": %u,"ZT_EOL_S - "\t\"revision\": %u,"ZT_EOL_S - "\t\"platform\": %d,"ZT_EOL_S - "\t\"architecture\": %d,"ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\","ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\""ZT_EOL_S + "\t\"timestamp\": %llu," ZT_EOL_S + "\t\"testId\": \"%.16llx\"," ZT_EOL_S + "\t\"upstream\": \"%.10llx\"," ZT_EOL_S + "\t\"current\": \"%.10llx\"," ZT_EOL_S + "\t\"receivedTimestamp\": %llu," ZT_EOL_S + "\t\"remoteTimestamp\": %llu," ZT_EOL_S + "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S + "\t\"flags\": %llu," ZT_EOL_S + "\t\"sourcePacketHopCount\": %u," ZT_EOL_S + "\t\"errorCode\": %u," ZT_EOL_S + "\t\"vendor\": %d," ZT_EOL_S + "\t\"protocolVersion\": %u," ZT_EOL_S + "\t\"majorVersion\": %u," ZT_EOL_S + "\t\"minorVersion\": %u," ZT_EOL_S + "\t\"revision\": %u," ZT_EOL_S + "\t\"platform\": %d," ZT_EOL_S + "\t\"architecture\": %d," ZT_EOL_S + "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S + "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S "}", ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), (unsigned long long)report->timestamp, diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp index 0d549abc..15e0968f 100644 --- a/controller/SqliteNetworkController.hpp +++ b/controller/SqliteNetworkController.hpp @@ -14,15 +14,6 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ */ #ifndef ZT_SQLITENETWORKCONTROLLER_HPP @@ -38,9 +29,11 @@ #include "../node/NetworkController.hpp" #include "../node/Mutex.hpp" -#include "../osdep/Thread.hpp" +#include "../node/Utils.hpp" + +#include "../osdep/OSUtils.hpp" -#include "../ext/offbase/offbase.hpp" +#include "../ext/json/json.hpp" namespace ZeroTier { @@ -82,10 +75,6 @@ public: std::string &responseBody, std::string &responseContentType); - // threadMain() for backup thread -- do not call directly - void threadMain() - throw(); - private: unsigned int _doCPGet( const std::vector &path, @@ -97,11 +86,57 @@ private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); + inline nlohmann::json _readJson(const std::string &path) + { + std::string buf; + if (OSUtils::readFile(path.c_str(),buf)) { + try { + return nlohmann::json::parse(buf); + } catch ( ... ) {} + } + return nlohmann::json::object(); + } + + inline bool _writeJson(const std::string &path,const nlohmann::json &obj) + { + std::string buf(obj.dump(2)); + return OSUtils::writeFile(path.c_str(),buf); + } + + inline std::string _networkBP(const uint64_t nwid,bool create) + { + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); + std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(tmp); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _networkJP(const uint64_t nwid,bool create) + { + return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); + } + inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) + { + std::string p(_networkBP(nwid,create)); + p.push_back(ZT_PATH_SEPARATOR); + p.append("member"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(member.toString()); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) + { + return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); + } + + // These are const after construction Node *const _node; - std::string _instanceId; - offbase _db; - Thread _dbCommitThread; - volatile bool _dbCommitThreadRun; + std::string _path; // Circuit tests outstanding struct _CircuitTestEntry @@ -110,11 +145,11 @@ private: std::string jsonResults; }; std::map< uint64_t,_CircuitTestEntry > _circuitTests; + Mutex _circuitTests_m; // Last request time by address, for rate limitation std::map< std::pair,uint64_t > _lastRequestTime; - - Mutex _lock; + Mutex _lastRequestTime_m; }; } // namespace ZeroTier diff --git a/ext/json/LICENSE.MIT b/ext/json/LICENSE.MIT new file mode 100644 index 00000000..e2ac4891 --- /dev/null +++ b/ext/json/LICENSE.MIT @@ -0,0 +1,22 @@ +The library is licensed under the MIT License +: + +Copyright (c) 2013-2016 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/json/README.md b/ext/json/README.md new file mode 100644 index 00000000..c0bb61b1 --- /dev/null +++ b/ext/json/README.md @@ -0,0 +1,511 @@ +[![JSON for Modern C++](https://raw.githubusercontent.com/nlohmann/json/master/doc/json.gif)](https://github.com/nlohmann/json/releases) + +[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) +[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk?svg=true)](https://ci.appveyor.com/project/nlohmann/json) +[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) +[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/p5o4znPnGHJpDVqN) +[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) +[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) +[![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) +[![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) + +## Design goals + +There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: + +- **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you know, what I mean. + +- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. + +- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. + +Other aspects were not so important to us: + +- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. + +- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). + +See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. + + +## Integration + +The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add + +```cpp +#include "json.hpp" + +// for convenience +using json = nlohmann::json; +``` + +to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). + +:beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. + + +## Examples + +Here are some examples to give you an idea how to use the class. + +Assume you want to create the JSON object + +```json +{ + "pi": 3.141, + "happy": true, + "name": "Niels", + "nothing": null, + "answer": { + "everything": 42 + }, + "list": [1, 0, 2], + "object": { + "currency": "USD", + "value": 42.99 + } +} +``` + +With the JSON class, you could write: + +```cpp +// create an empty structure (null) +json j; + +// add a number that is stored as double (note the implicit conversion of j to an object) +j["pi"] = 3.141; + +// add a Boolean that is stored as bool +j["happy"] = true; + +// add a string that is stored as std::string +j["name"] = "Niels"; + +// add another null object by passing nullptr +j["nothing"] = nullptr; + +// add an object inside the object +j["answer"]["everything"] = 42; + +// add an array that is stored as std::vector (using an initializer list) +j["list"] = { 1, 0, 2 }; + +// add another object (using an initializer list of pairs) +j["object"] = { {"currency", "USD"}, {"value", 42.99} }; + +// instead, you could also write (which looks very similar to the JSON above) +json j2 = { + {"pi", 3.141}, + {"happy", true}, + {"name", "Niels"}, + {"nothing", nullptr}, + {"answer", { + {"everything", 42} + }}, + {"list", {1, 0, 2}}, + {"object", { + {"currency", "USD"}, + {"value", 42.99} + }} +}; +``` + +Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: + +```cpp +// a way to express the empty array [] +json empty_array_explicit = json::array(); + +// ways to express the empty object {} +json empty_object_implicit = json({}); +json empty_object_explicit = json::object(); + +// a way to express an _array_ of key/value pairs [["currency", "USD"], ["value", 42.99]] +json array_not_object = { json::array({"currency", "USD"}), json::array({"value", 42.99}) }; +``` + + +### Serialization / Deserialization + +You can create an object (deserialization) by appending `_json` to a string literal: + +```cpp +// create object from string literal +json j = "{ \"happy\": true, \"pi\": 3.141 }"_json; + +// or even nicer with a raw string literal +auto j2 = R"( + { + "happy": true, + "pi": 3.141 + } +)"_json; + +// or explicitly +auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); +``` + +You can also get a string representation (serialize): + +```cpp +// explicit conversion to string +std::string s = j.dump(); // {\"happy\":true,\"pi\":3.141} + +// serialization with pretty printing +// pass in the amount of spaces to indent +std::cout << j.dump(4) << std::endl; +// { +// "happy": true, +// "pi": 3.141 +// } +``` + +You can also use streams to serialize and deserialize: + +```cpp +// deserialize from standard input +json j; +std::cin >> j; + +// serialize to standard output +std::cout << j; + +// the setw manipulator was overloaded to set the indentation for pretty printing +std::cout << std::setw(4) << j << std::endl; +``` + +These operators work for any subclasses of `std::istream` or `std::ostream`. + +Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. + + +### STL-like access + +We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. + +```cpp +// create an array using push_back +json j; +j.push_back("foo"); +j.push_back(1); +j.push_back(true); + +// iterate the array +for (json::iterator it = j.begin(); it != j.end(); ++it) { + std::cout << *it << '\n'; +} + +// range-based for +for (auto& element : j) { + std::cout << element << '\n'; +} + +// getter/setter +const std::string tmp = j[0]; +j[1] = 42; +bool foo = j.at(2); + +// other stuff +j.size(); // 3 entries +j.empty(); // false +j.type(); // json::value_t::array +j.clear(); // the array is empty again + +// convenience type checkers +j.is_null(); +j.is_boolean(); +j.is_number(); +j.is_object(); +j.is_array(); +j.is_string(); + +// comparison +j == "[\"foo\", 1, true]"_json; // true + +// create an object +json o; +o["foo"] = 23; +o["bar"] = false; +o["baz"] = 3.141; + +// special iterator member functions for objects +for (json::iterator it = o.begin(); it != o.end(); ++it) { + std::cout << it.key() << " : " << it.value() << "\n"; +} + +// find an entry +if (o.find("foo") != o.end()) { + // there is an entry with key "foo" +} + +// or simpler using count() +int foo_present = o.count("foo"); // 1 +int fob_present = o.count("fob"); // 0 + +// delete an entry +o.erase("foo"); +``` + + +### Conversion from STL containers + +Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends how the elements are ordered in the respective STL container. + +```cpp +std::vector c_vector {1, 2, 3, 4}; +json j_vec(c_vector); +// [1, 2, 3, 4] + +std::deque c_deque {1.2, 2.3, 3.4, 5.6}; +json j_deque(c_deque); +// [1.2, 2.3, 3.4, 5.6] + +std::list c_list {true, true, false, true}; +json j_list(c_list); +// [true, true, false, true] + +std::forward_list c_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; +json j_flist(c_flist); +// [12345678909876, 23456789098765, 34567890987654, 45678909876543] + +std::array c_array {{1, 2, 3, 4}}; +json j_array(c_array); +// [1, 2, 3, 4] + +std::set c_set {"one", "two", "three", "four", "one"}; +json j_set(c_set); // only one entry for "one" is used +// ["four", "one", "three", "two"] + +std::unordered_set c_uset {"one", "two", "three", "four", "one"}; +json j_uset(c_uset); // only one entry for "one" is used +// maybe ["two", "three", "four", "one"] + +std::multiset c_mset {"one", "two", "one", "four"}; +json j_mset(c_mset); // only one entry for "one" is used +// maybe ["one", "two", "four"] + +std::unordered_multiset c_umset {"one", "two", "one", "four"}; +json j_umset(c_umset); // both entries for "one" are used +// maybe ["one", "two", "one", "four"] +``` + +Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. + +```cpp +std::map c_map { {"one", 1}, {"two", 2}, {"three", 3} }; +json j_map(c_map); +// {"one": 1, "three": 3, "two": 2 } + +std::unordered_map c_umap { {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; +json j_umap(c_umap); +// {"one": 1.2, "two": 2.3, "three": 3.4} + +std::multimap c_mmap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; +json j_mmap(c_mmap); // only one entry for key "three" is used +// maybe {"one": true, "two": true, "three": true} + +std::unordered_multimap c_ummap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; +json j_ummap(c_ummap); // only one entry for key "three" is used +// maybe {"one": true, "two": true, "three": true} +``` + +### JSON Pointer and JSON Patch + +The library supports **JSON Pointer** ([RFC 6901](https://tools.ietf.org/html/rfc6901)) as alternative means to address structured values. On top of this, **JSON Patch** ([RFC 6902](https://tools.ietf.org/html/rfc6902)) allows to describe differences between two JSON values - effectively allowing patch and diff operations known from Unix. + +```cpp +// a JSON value +json j_original = R"({ + "baz": ["one", "two", "three"], + "foo": "bar" +})"_json; + +// access members with a JSON pointer (RFC 6901) +j_original["/baz/1"_json_pointer]; +// "two" + +// a JSON patch (RFC 6902) +json j_patch = R"([ + { "op": "replace", "path": "/baz", "value": "boo" }, + { "op": "add", "path": "/hello", "value": ["world"] }, + { "op": "remove", "path": "/foo"} +])"_json; + +// apply the patch +json j_result = j_original.patch(j_patch); +// { +// "baz": "boo", +// "hello": ["world"] +// } + +// calculate a JSON patch from two JSON values +json::diff(j_result, j_original); +// [ +// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, +// { "op": "remove","path": "/hello" }, +// { "op": "add", "path": "/foo", "value": "bar" } +// ] +``` + + +### Implicit conversions + +The type of the JSON object is determined automatically by the expression to store. Likewise, the stored value is implicitly converted. + +```cpp +// strings +std::string s1 = "Hello, world!"; +json js = s1; +std::string s2 = js; + +// Booleans +bool b1 = true; +json jb = b1; +bool b2 = jb; + +// numbers +int i = 42; +json jn = i; +double f = jn; + +// etc. +``` + +You can also explicitly ask for the value: + +```cpp +std::string vs = js.get(); +bool vb = jb.get(); +int vi = jn.get(); + +// etc. +``` + + +## Supported compilers + +Though it's 2016 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: + +- GCC 4.9 - 6.0 (and possibly later) +- Clang 3.4 - 3.9 (and possibly later) +- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) + +I would be happy to learn about other compilers/versions. + +Please note: + +- GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. +- Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. + + ``` + APP_STL := c++_shared + NDK_TOOLCHAIN_VERSION := clang3.6 + APP_CPPFLAGS += -frtti -fexceptions + ``` + + The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. + +- For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). + +The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): + +| Compiler | Operating System | Version String | +|-----------------|------------------------------|----------------| +| GCC 4.9.3 | Ubuntu 14.04.4 LTS | g++-4.9 (Ubuntu 4.9.3-8ubuntu2~14.04) 4.9.3 | +| GCC 5.3.0 | Ubuntu 14.04.4 LTS | g++-5 (Ubuntu 5.3.0-3ubuntu1~14.04) 5.3.0 20151204 | +| GCC 6.1.1 | Ubuntu 14.04.4 LTS | g++-6 (Ubuntu 6.1.1-3ubuntu11~14.04.1) 6.1.1 20160511 | +| Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | +| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | +| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | +| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | +| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | +| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | +| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | +| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | +| Clang Xcode 8.0 | Darwin Kernel Version 15.5.0 (OSX 10.11.5) | Apple LLVM version 8.0.0 (clang-800.0.24.1) | +| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | + + +## License + + + +The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): + +Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +## Thanks + +I deeply appreciate the help of the following people. + +- [Teemperor](https://github.com/Teemperor) implemented CMake support and lcov integration, realized escape and Unicode handling in the string parser, and fixed the JSON serialization. +- [elliotgoodrich](https://github.com/elliotgoodrich) fixed an issue with double deletion in the iterator classes. +- [kirkshoop](https://github.com/kirkshoop) made the iterators of the class composable to other libraries. +- [wancw](https://github.com/wanwc) fixed a bug that hindered the class to compile with Clang. +- Tomas Åblad found a bug in the iterator implementation. +- [Joshua C. Randall](https://github.com/jrandall) fixed a bug in the floating-point serialization. +- [Aaron Burghardt](https://github.com/aburgh) implemented code to parse streams incrementally. Furthermore, he greatly improved the parser class by allowing the definition of a filter function to discard undesired elements while parsing. +- [Daniel Kopeček](https://github.com/dkopecek) fixed a bug in the compilation with GCC 5.0. +- [Florian Weber](https://github.com/Florianjw) fixed a bug in and improved the performance of the comparison operators. +- [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. +- [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. +- [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. +- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. +- [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. +- [dariomt](https://github.com/dariomt) fixed some typos in the examples. +- [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. +- [Colin Hirsch](https://github.com/ColinH) took care of a small namespace issue. +- [Huu Nguyen](https://github.com/whoshuu) correct a variable name in the documentation. +- [Silverweed](https://github.com/silverweed) overloaded `parse()` to accept an rvalue reference. +- [dariomt](https://github.com/dariomt) fixed a subtlety in MSVC type support and implemented the `get_ref()` function to get a reference to stored values. +- [ZahlGraf](https://github.com/ZahlGraf) added a workaround that allows compilation using Android NDK. +- [whackashoe](https://github.com/whackashoe) replaced a function that was marked as unsafe by Visual Studio. +- [406345](https://github.com/406345) fixed two small warnings. +- [Glen Fernandes](https://github.com/glenfe) noted a potential portability problem in the `has_mapped_type` function. +- [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. +- [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. +- [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. +- [msm-](https://github.com/msm-) added support for american fuzzy lop. +- [Annihil](https://github.com/Annihil) fixed an example in the README file. +- [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. +- [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. +- [abc100m](https://github.com/abc100m) analyzed the issues with GCC 4.8 and proposed a [partial solution](https://github.com/nlohmann/json/pull/212). +- [zewt](https://github.com/zewt) added useful notes to the README file about Android. +- [Róbert Márki](https://github.com/robertmrk) added a fix to use move iterators and improved the integration via CMake. +- [Chris Kitching](https://github.com/ChrisKitching) cleaned up the CMake files. +- [Tom Needham](https://github.com/06needhamt) fixed a subtle bug with MSVC 2015 which was also proposed by [Michael K.](https://github.com/Epidal). +- [Mário Feroldi](https://github.com/thelostt) fixed a small typo. +- [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. +- [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. + +Thanks a lot for helping out! + + +## Notes + +- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). +- As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. + + +## Execute unit tests + +To compile and run the tests, you need to execute + +```sh +$ make +$ ./json_unit "*" + +=============================================================================== +All tests passed (8905012 assertions in 32 test cases) +``` + +For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/ext/json/json.hpp b/ext/json/json.hpp new file mode 100644 index 00000000..878fb899 --- /dev/null +++ b/ext/json/json.hpp @@ -0,0 +1,10435 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 2.0.2 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +Copyright (c) 2013-2016 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// exclude unsupported compilers +#if defined(__clang__) + #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) + #if CLANG_VERSION < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#elif defined(__GNUC__) + #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + #if GCC_VERSION < 40900 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + + +/*! +@brief unnamed namespace with internal helper functions +@since version 1.0.0 +*/ +namespace +{ +/*! +@brief Helper to determine whether there's a key_type for T. + +Thus helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. + +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0 +*/ +template +struct has_mapped_type +{ + private: + template static char test(typename C::mapped_type*); + template static char (&test(...))[2]; + public: + static constexpr bool value = sizeof(test(0)) == 1; +}; + +/*! +@brief helper class to create locales with decimal point + +This struct is used a default locale during the JSON serialization. JSON +requires the decimal point to be `.`, so this function overloads the +`do_decimal_point()` function to return `.`. This function is called by +float-to-string conversions to retrieve the decimal separator between integer +and fractional parts. + +@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 +@since version 2.0.0 +*/ +struct DecimalSeparator : std::numpunct +{ + char do_decimal_point() const + { + return '.'; + } +}; + +} + +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the class + has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. + +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 + +@nosubgrouping +*/ +template < + template class ObjectType = std::map, + template class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator + > +class basic_json +{ + private: + /// workaround type for MSVC + using basic_json_t = basic_json; + + public: + // forward declarations + template class json_reverse_iterator; + class json_pointer; + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + class iterator; + /// a const iterator for a basic_json container + class const_iterator; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() + { + return allocator_type(); + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /*! + @brief a type for an object + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. + + To store objects in C++, a type is defined by the template parameters + described below. + + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) + + #### Default type + + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: + + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less, // key_compare + std::allocator> // allocator_type + > + @endcode + + #### Behavior + + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: + + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType, + AllocatorType>>; + + /*! + @brief a type for an array + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. + + To store objects in C++, a type is defined by the template parameters + explained below. + + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + + #### Default type + + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: + + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator // allocator_type + > + @endcode + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. + + #### Storage + + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value + + @since version 1.0.0 + */ + using array_t = ArrayType>; + + /*! + @brief a type for a string + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. + + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. + + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. + + #### Default type + + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: + + @code {.cpp} + std::string + @endcode + + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage + + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. + + @since version 1.0.0 + */ + using string_t = StringType; + + /*! + @brief a type for a boolean + + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. + + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. + + #### Default type + + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. + + @since version 1.0.0 + */ + using boolean_t = BooleanType; + + /*! + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. + + #### Default type + + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: + + @code {.cpp} + int64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_integer_t = NumberIntegerType; + + /*! + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. + + #### Default type + + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: + + @code {.cpp} + uint64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) + + @since version 2.0.0 + */ + using number_unsigned_t = NumberUnsignedType; + + /*! + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. + + #### Default type + + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: + + @code {.cpp} + double + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. + + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. + + #### Storage + + Floating-point number values are stored directly inside a @ref basic_json + type. + + @sa @ref number_integer_t -- type for number values (integer) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_float_t = NumberFloatType; + + /// @} + + + /////////////////////////// + // JSON type enumeration // + /////////////////////////// + + /*! + @brief the JSON type enumeration + + This enumeration collects the different JSON types. It is internally used + to distinguish the stored values, and the functions @ref is_null(), @ref + is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref + is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and + @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and + @ref is_structured() rely on it. + + @note There are three enumeration entries (number_integer, + number_unsigned, and number_float), because the library distinguishes + these three types for numbers: @ref number_unsigned_t is used for unsigned + integers, @ref number_integer_t is used for signed integers, and @ref + number_float_t is used for floating-point numbers or to approximate + integers which do not fit in the limits of their respective type. + + @sa @ref basic_json(const value_t value_type) -- create a JSON value with + the default value for a given type + + @since version 1.0.0 + */ + enum class value_t : uint8_t + { + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function + }; + + + private: + + /// helper for exception-safe object creation + template + static T* create(Args&& ... args) + { + AllocatorType alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward(args)...); + assert(object.get() != nullptr); + return object.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } + + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } + + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } + + default: + { + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) + { + string = create(value); + } + + /// constructor for objects + json_value(const object_t& value) + { + object = create(value); + } + + /// constructor for arrays + json_value(const array_t& value) + { + array = create(value); + } + }; + + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + */ + void assert_invariant() const + { + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /*! + @brief JSON callback events + + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. + + @image html callback_events.png "Example when certain parse events are triggered" + + @since version 1.0.0 + */ + enum class parse_event_t : uint8_t + { + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; + + /*! + @brief per-element parser callback type + + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. + + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. + + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: + + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. + + @param[in] depth the depth of the recursion during parsing + + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called + + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events + + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. + + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const string_t&, parser_callback_t) for examples + + @since version 1.0.0 + */ + using parser_callback_t = std::function; + + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /*! + @brief create an empty value with a given type + + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @param[in] value_type the type of the value to create + + @complexity Constant. + + @throw std::bad_alloc if allocation for object, array, or string value + fails + + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + @sa @ref basic_json(boolean_t value) -- create a boolean value + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const object_t&) -- create a object value + @sa @ref basic_json(const array_t&) -- create a array value + @sa @ref basic_json(const number_float_t) -- create a number + (floating-point) value + @sa @ref basic_json(const number_integer_t) -- create a number (integer) + value + @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) + value + + @since version 1.0.0 + */ + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) + { + assert_invariant(); + } + + /*! + @brief create a null object (implicitly) + + Create a `null` JSON value. This is the implicit version of the `null` + value constructor as it takes no parameters. + + @note The class invariant is satisfied, because it poses no requirements + for null values. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - As postcondition, it holds: `basic_json().empty() == true`. + + @liveexample{The following code shows the constructor for a `null` JSON + value.,basic_json} + + @sa @ref basic_json(std::nullptr_t) -- create a `null` value + + @since version 1.0.0 + */ + basic_json() = default; + + /*! + @brief create a null object (explicitly) + + Create a `null` JSON value. This is the explicitly version of the `null` + value constructor as it takes a null pointer as parameter. It allows to + create `null` values by explicitly assigning a `nullptr` to a JSON value. + The passed null pointer itself is not read -- it is only used to choose + the right constructor. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @liveexample{The following code shows the constructor with null pointer + parameter.,basic_json__nullptr_t} + + @sa @ref basic_json() -- default constructor (implicitly creating a `null` + value) + + @since version 1.0.0 + */ + basic_json(std::nullptr_t) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /*! + @brief create an object (explicit) + + Create an object JSON value with a given content. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with an @ref + object_t parameter.,basic_json__object_t} + + @sa @ref basic_json(const CompatibleObjectType&) -- create an object value + from a compatible STL container + + @since version 1.0.0 + */ + basic_json(const object_t& val) + : m_type(value_t::object), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an object (implicit) + + Create an object JSON value with a given content. This constructor allows + any type @a CompatibleObjectType that can be used to construct values of + type @ref object_t. + + @tparam CompatibleObjectType An object type whose `key_type` and + `value_type` is compatible to @ref object_t. Examples include `std::map`, + `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with + a `key_type` of `std::string`, and a `value_type` from which a @ref + basic_json value can be constructed. + + @param[in] val a value for the object + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for object value fails + + @liveexample{The following code shows the constructor with several + compatible object type parameters.,basic_json__CompatibleObjectType} + + @sa @ref basic_json(const object_t&) -- create an object value + + @since version 1.0.0 + */ + template ::value and + std::is_constructible::value, int>::type + = 0> + basic_json(const CompatibleObjectType& val) + : m_type(value_t::object) + { + using std::begin; + using std::end; + m_value.object = create(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create an array (explicit) + + Create an array JSON value with a given content. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with an @ref array_t + parameter.,basic_json__array_t} + + @sa @ref basic_json(const CompatibleArrayType&) -- create an array value + from a compatible STL containers + + @since version 1.0.0 + */ + basic_json(const array_t& val) + : m_type(value_t::array), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an array (implicit) + + Create an array JSON value with a given content. This constructor allows + any type @a CompatibleArrayType that can be used to construct values of + type @ref array_t. + + @tparam CompatibleArrayType An object type whose `value_type` is + compatible to @ref array_t. Examples include `std::vector`, `std::deque`, + `std::list`, `std::forward_list`, `std::array`, `std::set`, + `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a + `value_type` from which a @ref basic_json value can be constructed. + + @param[in] val a value for the array + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for array value fails + + @liveexample{The following code shows the constructor with several + compatible array type parameters.,basic_json__CompatibleArrayType} + + @sa @ref basic_json(const array_t&) -- create an array value + + @since version 1.0.0 + */ + template ::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + std::is_constructible::value, int>::type + = 0> + basic_json(const CompatibleArrayType& val) + : m_type(value_t::array) + { + using std::begin; + using std::end; + m_value.array = create(begin(val), end(val)); + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create an string JSON value with a given content. + + @param[in] val a value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with an @ref + string_t parameter.,basic_json__string_t} + + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const string_t& val) + : m_type(value_t::string), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create a string (explicit) + + Create a string JSON value with a given content. + + @param[in] val a literal value for the string + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the constructor with string literal + parameter.,basic_json__string_t_value_type} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const CompatibleStringType&) -- create a string value + from a compatible string container + + @since version 1.0.0 + */ + basic_json(const typename string_t::value_type* val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a string (implicit) + + Create a string JSON value with a given content. + + @param[in] val a value for the string + + @tparam CompatibleStringType an string type which is compatible to @ref + string_t, for instance `std::string`. + + @complexity Linear in the size of the passed @a val. + + @throw std::bad_alloc if allocation for string value fails + + @liveexample{The following code shows the construction of a string value + from a compatible type.,basic_json__CompatibleStringType} + + @sa @ref basic_json(const string_t&) -- create a string value + @sa @ref basic_json(const typename string_t::value_type*) -- create a + string value from a character pointer + + @since version 1.0.0 + */ + template ::value, int>::type + = 0> + basic_json(const CompatibleStringType& val) + : basic_json(string_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a boolean (explicit) + + Creates a JSON boolean type from a given value. + + @param[in] val a boolean value to store + + @complexity Constant. + + @liveexample{The example below demonstrates boolean + values.,basic_json__boolean_t} + + @since version 1.0.0 + */ + basic_json(boolean_t val) noexcept + : m_type(value_t::boolean), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number (explicit) + + Create an integer number JSON value with a given content. + + @tparam T A helper type to remove this function via SFINAE in case @ref + number_integer_t is the same as `int`. In this case, this constructor + would have the same signature as @ref basic_json(const int value). Note + the helper type @a T is not visible in this constructor's interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value.,basic_json__number_integer_t} + + @sa @ref basic_json(const int) -- create a number value (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + template::value) + and std::is_same::value + , int>::type + = 0> + basic_json(const number_integer_t val) noexcept + : m_type(value_t::number_integer), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an integer number from an enum type (explicit) + + Create an integer number JSON value with a given content. + + @param[in] val an integer to create a JSON number from + + @note This constructor allows to pass enums directly to a constructor. As + C++ has no way of specifying the type of an anonymous enum explicitly, we + can only rely on the fact that such values implicitly convert to int. As + int may already be the same type of number_integer_t, we may need to + switch off the constructor @ref basic_json(const number_integer_t). + + @complexity Constant. + + @liveexample{The example below shows the construction of an integer + number value from an anonymous enum.,basic_json__const_int} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number + value (integer) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const int val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create an integer number (implicit) + + Create an integer number JSON value with a given content. This constructor + allows any type @a CompatibleNumberIntegerType that can be used to + construct values of type @ref number_integer_t. + + @tparam CompatibleNumberIntegerType An integer type which is compatible to + @ref number_integer_t. Examples include the types `int`, `int32_t`, + `long`, and `short`. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @liveexample{The example below shows the construction of several integer + number values from compatible + types.,basic_json__CompatibleIntegerNumberType} + + @sa @ref basic_json(const number_integer_t) -- create a number value + (integer) + @sa @ref basic_json(const int) -- create a number value (integer) + + @since version 1.0.0 + */ + template::value and + std::numeric_limits::is_integer and + std::numeric_limits::is_signed, + CompatibleNumberIntegerType>::type + = 0> + basic_json(const CompatibleNumberIntegerType val) noexcept + : m_type(value_t::number_integer), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create an unsigned integer number (explicit) + + Create an unsigned integer number JSON value with a given content. + + @tparam T helper type to compare number_unsigned_t and unsigned int (not + visible in) the interface. + + @param[in] val an integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number + value (unsigned integer) from a compatible number type + + @since version 2.0.0 + */ + template::value) + and std::is_same::value + , int>::type + = 0> + basic_json(const number_unsigned_t val) noexcept + : m_type(value_t::number_unsigned), m_value(val) + { + assert_invariant(); + } + + /*! + @brief create an unsigned number (implicit) + + Create an unsigned number JSON value with a given content. This + constructor allows any type @a CompatibleNumberUnsignedType that can be + used to construct values of type @ref number_unsigned_t. + + @tparam CompatibleNumberUnsignedType An integer type which is compatible + to @ref number_unsigned_t. Examples may include the types `unsigned int`, + `uint32_t`, or `unsigned short`. + + @param[in] val an unsigned integer to create a JSON number from + + @complexity Constant. + + @sa @ref basic_json(const number_unsigned_t) -- create a number value + (unsigned) + + @since version 2.0.0 + */ + template ::value and + std::numeric_limits::is_integer and + not std::numeric_limits::is_signed, + CompatibleNumberUnsignedType>::type + = 0> + basic_json(const CompatibleNumberUnsignedType val) noexcept + : m_type(value_t::number_unsigned), + m_value(static_cast(val)) + { + assert_invariant(); + } + + /*! + @brief create a floating-point number (explicit) + + Create a floating-point number JSON value with a given content. + + @param[in] val a floating-point value to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is created + instead. + + @complexity Constant. + + @liveexample{The following example creates several floating-point + values.,basic_json__number_float_t} + + @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number + value (floating-point) from a compatible number type + + @since version 1.0.0 + */ + basic_json(const number_float_t val) noexcept + : m_type(value_t::number_float), m_value(val) + { + // replace infinity and NAN by null + if (not std::isfinite(val)) + { + m_type = value_t::null; + m_value = json_value(); + } + + assert_invariant(); + } + + /*! + @brief create an floating-point number (implicit) + + Create an floating-point number JSON value with a given content. This + constructor allows any type @a CompatibleNumberFloatType that can be used + to construct values of type @ref number_float_t. + + @tparam CompatibleNumberFloatType A floating-point type which is + compatible to @ref number_float_t. Examples may include the types `float` + or `double`. + + @param[in] val a floating-point to create a JSON number from + + @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 + disallows NaN values: + > Numeric values that cannot be represented in the grammar below (such as + > Infinity and NaN) are not permitted. + In case the parameter @a val is not a number, a JSON null value is + created instead. + + @complexity Constant. + + @liveexample{The example below shows the construction of several + floating-point number values from compatible + types.,basic_json__CompatibleNumberFloatType} + + @sa @ref basic_json(const number_float_t) -- create a number value + (floating-point) + + @since version 1.0.0 + */ + template::value and + std::is_floating_point::value>::type + > + basic_json(const CompatibleNumberFloatType val) noexcept + : basic_json(number_float_t(val)) + { + assert_invariant(); + } + + /*! + @brief create a container (array or object) from an initializer list + + Creates a JSON value of type array or object from the passed initializer + list @a init. In case @a type_deduction is `true` (default), the type of + the JSON value to be created is deducted from the initializer list @a init + according to the following rules: + + 1. If the list is empty, an empty JSON object value `{}` is created. + 2. If the list consists of pairs whose first element is a string, a JSON + object value is created where the first elements of the pairs are + treated as keys and the second elements are as values. + 3. In all other cases, an array is created. + + The rules aim to create the best fit between a C++ initializer list and + JSON values. The rationale is as follows: + + 1. The empty initializer list is written as `{}` which is exactly an empty + JSON object. + 2. C++ has now way of describing mapped types other than to list a list of + pairs. As JSON requires that keys must be of type string, rule 2 is the + weakest constraint one can pose on initializer lists to interpret them + as an object. + 3. In all other cases, the initializer list could not be interpreted as + JSON object type, so interpreting it as JSON array type is safe. + + With the rules described above, the following JSON values cannot be + expressed by an initializer list: + + - the empty array (`[]`): use @ref array(std::initializer_list) + with an empty initializer list in this case + - arrays whose elements satisfy rule 2: use @ref + array(std::initializer_list) with the same initializer list + in this case + + @note When used without parentheses around an empty initializer list, @ref + basic_json() is called instead of this function, yielding the JSON null + value. + + @param[in] init initializer list with JSON values + + @param[in] type_deduction internal parameter; when set to `true`, the type + of the JSON value is deducted from the initializer list @a init; when set + to `false`, the type provided via @a manual_type is forced. This mode is + used by the functions @ref array(std::initializer_list) and + @ref object(std::initializer_list). + + @param[in] manual_type internal parameter; when @a type_deduction is set + to `false`, the created JSON value will use the provided type (only @ref + value_t::array and @ref value_t::object are valid); when @a type_deduction + is set to `true`, this parameter has no effect + + @throw std::domain_error if @a type_deduction is `false`, @a manual_type + is `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string; example: `"cannot create object from + initializer list"` + + @complexity Linear in the size of the initializer list @a init. + + @liveexample{The example below shows how JSON values are created from + initializer lists.,basic_json__list_init_t} + + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + basic_json(std::initializer_list init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const basic_json & element) + { + return element.is_array() and element.size() == 2 and element[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (not type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (manual_type == value_t::object and not is_an_object) + { + throw std::domain_error("cannot create object from initializer list"); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + std::for_each(init.begin(), init.end(), [this](const basic_json & element) + { + m_value.object->emplace(*(element[0].m_value.string), element[1]); + }); + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init); + } + + assert_invariant(); + } + + /*! + @brief explicitly create an array from an initializer list + + Creates a JSON array value from a given initializer list. That is, given a + list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the + initializer list is empty, the empty array `[]` is created. + + @note This function is only needed to express two edge cases that cannot + be realized with the initializer list constructor (@ref + basic_json(std::initializer_list, bool, value_t)). These cases + are: + 1. creating an array whose elements are all pairs whose first element is a + string -- in this case, the initializer list constructor would create an + object, taking the first elements as keys + 2. creating an empty array -- passing the empty initializer list to the + initializer list constructor yields an empty object + + @param[in] init initializer list with JSON values to create an array from + (optional) + + @return JSON array value + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `array` + function.,array} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + static basic_json array(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::array); + } + + /*! + @brief explicitly create an object from an initializer list + + Creates a JSON object value from a given initializer list. The initializer + lists elements must be pairs, and their first elements must be strings. If + the initializer list is empty, the empty object `{}` is created. + + @note This function is only added for symmetry reasons. In contrast to the + related function @ref array(std::initializer_list), there are + no cases which can only be expressed by this function. That is, any + initializer list @a init can also be passed to the initializer list + constructor @ref basic_json(std::initializer_list, bool, + value_t). + + @param[in] init initializer list to create an object from (optional) + + @return JSON object value + + @throw std::domain_error if @a init is not a pair whose first elements are + strings; thrown by + @ref basic_json(std::initializer_list, bool, value_t) + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `object` + function.,object} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + + @since version 1.0.0 + */ + static basic_json object(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::object); + } + + /*! + @brief construct an array with count copies of given value + + Constructs a JSON array value by creating @a cnt copies of a passed value. + In case @a cnt is `0`, an empty array is created. As postcondition, + `std::distance(begin(),end()) == cnt` holds. + + @param[in] cnt the number of JSON copies of @a val to create + @param[in] val the JSON value to copy + + @complexity Linear in @a cnt. + + @liveexample{The following code shows examples for the @ref + basic_json(size_type\, const basic_json&) + constructor.,basic_json__size_type_basic_json} + + @since version 1.0.0 + */ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + assert_invariant(); + } + + /*! + @brief construct a JSON container given an iterator range + + Constructs the JSON value with the contents of the range `[first, last)`. + The semantics depends on the different types a JSON value can have: + - In case of primitive types (number, boolean, or string), @a first must + be `begin()` and @a last must be `end()`. In this case, the value is + copied. Otherwise, std::out_of_range is thrown. + - In case of structured types (array, object), the constructor behaves as + similar versions for `std::vector`. + - In case of a null type, std::domain_error is thrown. + + @tparam InputIT an input iterator type (@ref iterator or @ref + const_iterator) + + @param[in] first begin of the range to copy from (included) + @param[in] last end of the range to copy from (excluded) + + @pre Iterators @a first and @a last must be initialized. + + @throw std::domain_error if iterators are not compatible; that is, do not + belong to the same JSON value; example: `"iterators are not compatible"` + @throw std::out_of_range if iterators are for a primitive type (number, + boolean, or string) where an out of range error can be detected easily; + example: `"iterators out of range"` + @throw std::bad_alloc if allocation for object, array, or string fails + @throw std::domain_error if called with a null value; example: `"cannot + use construct with iterators from null"` + + @complexity Linear in distance between @a first and @a last. + + @liveexample{The example below shows several ways to create JSON values by + specifying a subrange with iterators.,basic_json__InputIt_InputIt} + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + basic_json(InputIT first, InputIT last) + { + assert(first.m_object != nullptr); + assert(last.m_object != nullptr); + + // make sure iterator fits the current value + if (first.m_object != last.m_object) + { + throw std::domain_error("iterators are not compatible"); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + throw std::out_of_range("iterators out of range"); + } + break; + } + + default: + { + break; + } + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); + } + } + + assert_invariant(); + } + + /*! + @brief construct a JSON value given an input stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates constructing a JSON value from + a `std::stringstream` with and without callback + function.,basic_json__istream} + + @since version 2.0.0 + */ + explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) + { + *this = parser(i, cb).parse(); + assert_invariant(); + } + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + /*! + @brief copy constructor + + Creates a copy of a given JSON value. + + @param[in] other the JSON value to copy + + @complexity Linear in the size of @a other. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - As postcondition, it holds: `other == basic_json(other)`. + + @throw std::bad_alloc if allocation for object, array, or string fails. + + @liveexample{The following code shows an example for the copy + constructor.,basic_json__basic_json} + + @since version 1.0.0 + */ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + default: + { + break; + } + } + + assert_invariant(); + } + + /*! + @brief move constructor + + Move constructor. Constructs a JSON value with the contents of the given + value @a other using move semantics. It "steals" the resources from @a + other and leaves it as JSON null value. + + @param[in,out] other value to move to this object + + @post @a other is a JSON null value + + @complexity Constant. + + @liveexample{The code below shows the move constructor explicitly called + via std::move.,basic_json__moveconstructor} + + @since version 1.0.0 + */ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + assert_invariant(); + } + + /*! + @brief copy assignment + + Copy assignment operator. Copies a JSON value via the "copy and swap" + strategy: It is expressed in terms of the copy constructor, destructor, + and the swap() member function. + + @param[in] other value to copy from + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + + @liveexample{The code below shows and example for the copy assignment. It + creates a copy of value `a` which is then swapped with `b`. Finally\, the + copy of `a` (which is the null value after the swap) is + destroyed.,basic_json__copyassignment} + + @since version 1.0.0 + */ + reference& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + assert_invariant(); + return *this; + } + + /*! + @brief destructor + + Destroys the JSON value and frees all allocated memory. + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - All stored elements are destroyed and all memory is freed. + + @since version 1.0.0 + */ + ~basic_json() + { + assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + AllocatorType alloc; + alloc.destroy(m_value.object); + alloc.deallocate(m_value.object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + alloc.destroy(m_value.array); + alloc.deallocate(m_value.array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + break; + } + + default: + { + // all other types need no specific destructor + break; + } + } + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /*! + @brief serialization + + Serialization function for JSON values. The function tries to mimic + Python's `json.dumps()` function, and currently supports its @a indent + parameter. + + @param[in] indent If indent is nonnegative, then array elements and object + members will be pretty-printed with that indent level. An indent level of + `0` will only insert newlines. `-1` (the default) selects the most compact + representation. + + @return string containing the serialization of the JSON value + + @complexity Linear. + + @liveexample{The following example shows the effect of different @a indent + parameters to the result of the serialization.,dump} + + @see https://docs.python.org/2/library/json.html#json.dump + + @since version 1.0.0 + */ + string_t dump(const int indent = -1) const + { + std::stringstream ss; + // fix locale problems + ss.imbue(std::locale(std::locale(), new DecimalSeparator)); + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits::digits10 + ss.precision(std::numeric_limits::digits10); + + if (indent >= 0) + { + dump(ss, true, static_cast(indent)); + } + else + { + dump(ss, false, 0); + } + + return ss.str(); + } + + /*! + @brief return the type of the JSON value (explicit) + + Return the type of the JSON value as a value from the @ref value_t + enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `type()` for all JSON + types.,type} + + @since version 1.0.0 + */ + constexpr value_t type() const noexcept + { + return m_type; + } + + /*! + @brief return whether type is primitive + + This function returns true iff the JSON type is primitive (string, number, + boolean, or null). + + @return `true` if type is primitive (string, number, boolean, or null), + `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_primitive()` for all JSON + types.,is_primitive} + + @sa @ref is_structured() -- returns whether JSON value is structured + @sa @ref is_null() -- returns whether JSON value is `null` + @sa @ref is_string() -- returns whether JSON value is a string + @sa @ref is_boolean() -- returns whether JSON value is a boolean + @sa @ref is_number() -- returns whether JSON value is a number + + @since version 1.0.0 + */ + constexpr bool is_primitive() const noexcept + { + return is_null() or is_string() or is_boolean() or is_number(); + } + + /*! + @brief return whether type is structured + + This function returns true iff the JSON type is structured (array or + object). + + @return `true` if type is structured (array or object), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_structured()` for all JSON + types.,is_structured} + + @sa @ref is_primitive() -- returns whether value is primitive + @sa @ref is_array() -- returns whether value is an array + @sa @ref is_object() -- returns whether value is an object + + @since version 1.0.0 + */ + constexpr bool is_structured() const noexcept + { + return is_array() or is_object(); + } + + /*! + @brief return whether value is null + + This function returns true iff the JSON value is null. + + @return `true` if type is null, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_null()` for all JSON + types.,is_null} + + @since version 1.0.0 + */ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /*! + @brief return whether value is a boolean + + This function returns true iff the JSON value is a boolean. + + @return `true` if type is boolean, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_boolean()` for all JSON + types.,is_boolean} + + @since version 1.0.0 + */ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /*! + @brief return whether value is a number + + This function returns true iff the JSON value is a number. This includes + both integer and floating-point values. + + @return `true` if type is number (regardless whether integer, unsigned + integer or floating-type), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number()` for all JSON + types.,is_number} + + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number() const noexcept + { + return is_number_integer() or is_number_float(); + } + + /*! + @brief return whether value is an integer number + + This function returns true iff the JSON value is an integer or unsigned + integer number. This excludes floating-point values. + + @return `true` if type is an integer or unsigned integer number, `false` + otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_integer()` for all + JSON types.,is_number_integer} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is an unsigned integer number + + This function returns true iff the JSON value is an unsigned integer + number. This excludes floating-point and (signed) integer values. + + @return `true` if type is an unsigned integer number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_unsigned()` for all + JSON types.,is_number_unsigned} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 2.0.0 + */ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is a floating-point number + + This function returns true iff the JSON value is a floating-point number. + This excludes integer and unsigned integer values. + + @return `true` if type is a floating-point number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_float()` for all + JSON types.,is_number_float} + + @sa @ref is_number() -- check if value is number + @sa @ref is_number_integer() -- check if value is an integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + + @since version 1.0.0 + */ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /*! + @brief return whether value is an object + + This function returns true iff the JSON value is an object. + + @return `true` if type is object, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_object()` for all JSON + types.,is_object} + + @since version 1.0.0 + */ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /*! + @brief return whether value is an array + + This function returns true iff the JSON value is an array. + + @return `true` if type is array, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_array()` for all JSON + types.,is_array} + + @since version 1.0.0 + */ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /*! + @brief return whether value is a string + + This function returns true iff the JSON value is a string. + + @return `true` if type is string, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_string()` for all JSON + types.,is_string} + + @since version 1.0.0 + */ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /*! + @brief return whether value is discarded + + This function returns true iff the JSON value was discarded during parsing + with a callback function (see @ref parser_callback_t). + + @note This function will always be `false` for JSON values after parsing. + That is, discarded values can only occur during parsing, but will be + removed when inside a structured value or replaced by null in other cases. + + @return `true` if type is discarded, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_discarded()` for all JSON + types.,is_discarded} + + @since version 1.0.0 + */ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /*! + @brief return the type of the JSON value (implicit) + + Implicitly return the type of the JSON value as a value from the @ref + value_t enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies the @ref value_t operator for + all JSON types.,operator__value_t} + + @since version 1.0.0 + */ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get an object (explicit) + template ::value and + std::is_convertible::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_object()) + { + return T(m_value.object->begin(), m_value.object->end()); + } + else + { + throw std::domain_error("type must be object, but is " + type_name()); + } + } + + /// get an object (explicit) + object_t get_impl(object_t*) const + { + if (is_object()) + { + return *(m_value.object); + } + else + { + throw std::domain_error("type must be object, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not std::is_same::value and + not std::is_arithmetic::value and + not std::is_convertible::value and + not has_mapped_type::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + T to_vector; + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get(); + }); + return to_vector; + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not std::is_same::value + , int>::type = 0> + std::vector get_impl(std::vector*) const + { + if (is_array()) + { + std::vector to_vector; + to_vector.reserve(m_value.array->size()); + std::transform(m_value.array->begin(), m_value.array->end(), + std::inserter(to_vector, to_vector.end()), [](basic_json i) + { + return i.get(); + }); + return to_vector; + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + template ::value and + not has_mapped_type::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_array()) + { + return T(m_value.array->begin(), m_value.array->end()); + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get an array (explicit) + array_t get_impl(array_t*) const + { + if (is_array()) + { + return *(m_value.array); + } + else + { + throw std::domain_error("type must be array, but is " + type_name()); + } + } + + /// get a string (explicit) + template ::value + , int>::type = 0> + T get_impl(T*) const + { + if (is_string()) + { + return *m_value.string; + } + else + { + throw std::domain_error("type must be string, but is " + type_name()); + } + } + + /// get a number (explicit) + template::value + , int>::type = 0> + T get_impl(T*) const + { + switch (m_type) + { + case value_t::number_integer: + { + return static_cast(m_value.number_integer); + } + + case value_t::number_unsigned: + { + return static_cast(m_value.number_unsigned); + } + + case value_t::number_float: + { + return static_cast(m_value.number_float); + } + + default: + { + throw std::domain_error("type must be number, but is " + type_name()); + } + } + } + + /// get a boolean (explicit) + constexpr boolean_t get_impl(boolean_t*) const + { + return is_boolean() + ? m_value.boolean + : throw std::domain_error("type must be boolean, but is " + type_name()); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t*) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t*) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t*) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t*) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t*) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t*) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This funcion helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw std::domain_error if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // helper type + using PointerType = typename std::add_pointer::type; + + // delegate the call to get_ptr<>() + auto ptr = obj.template get_ptr(); + + if (ptr != nullptr) + { + return *ptr; + } + else + { + throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + + obj.type_name()); + } + } + + public: + + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON; example: `"type must be object, but is null"` + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @internal + The idea of using a casted null pointer to choose the correct + implementation is from . + @endinternal + + @sa @ref operator ValueType() const for implicit conversion + @sa @ref get() for pointer-member access + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + ValueType get() const + { + return get_impl(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + PointerType get() noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value + , int>::type = 0> + constexpr const PointerType get() const noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are + made. + + @warning Writing data to the pointee of the result yields an undefined + state. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get_ptr} + + @since version 1.0.0 + */ + template::value + , int>::type = 0> + PointerType get_ptr() noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (implicit) + @copydoc get_ptr() + */ + template::value + and std::is_const::type>::value + , int>::type = 0> + constexpr const PointerType get_ptr() const noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a reference value (implicit) + + Implict reference access to the internally stored JSON value. No copies + are made. + + @warning Writing data to the referee of the result yields an undefined + state. + + @tparam ReferenceType reference type; must be a reference to @ref array_t, + @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or + @ref number_float_t. Enforced by static assertion. + + @return reference to the internally stored JSON value if the requested + reference type @a ReferenceType fits to the JSON value; throws + std::domain_error otherwise + + @throw std::domain_error in case passed type @a ReferenceType is + incompatible with the stored JSON value + + @complexity Constant. + + @liveexample{The example shows several calls to `get_ref()`.,get_ref} + + @since version 1.1.0 + */ + template::value + , int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a reference value (implicit) + @copydoc get_ref() + */ + template::value + and std::is_const::type>::value + , int>::type = 0> + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON, thrown by @ref get() const + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename + std::enable_if < + not std::is_pointer::value + and not std::is_same::value +#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 + and not std::is_same>::value +#endif + , int >::type = 0 > + operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /*! + @brief access specified array element with bounds checking + + Returns a reference to the element at specified location @a idx, with + bounds checking. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read and + written using `at()`.,at__size_type} + + @since version 1.0.0 + */ + reference at(size_type idx) + { + // at only works for arrays + if (is_array()) + { + try + { + return m_value.array->at(idx); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element with bounds checking + + Returns a const reference to the element at specified location @a idx, + with bounds checking. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + `at()`.,at__size_type_const} + + @since version 1.0.0 + */ + const_reference at(size_type idx) const + { + // at only works for arrays + if (is_array()) + { + try + { + return m_value.array->at(idx); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a reference to the element at with specified key @a key, with + bounds checking. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using `at()`.,at__object_t_key_type} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (is_object()) + { + try + { + return m_value.object->at(key); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("key '" + key + "' not found"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a const reference to the element at with specified key @a key, + with bounds checking. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + `at()`.,at__object_t_key_type_const} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (is_object()) + { + try + { + return m_value.object->at(key); + } + catch (std::out_of_range&) + { + // create better exception explanation + throw std::out_of_range("key '" + key + "' not found"); + } + } + else + { + throw std::domain_error("cannot use at() with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a reference to the element at specified location @a idx. + + @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), + then the array is silently filled up with `null` values to make `idx` a + valid reference to the last stored element. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array or null; example: + `"cannot use operator[] with string"` + + @complexity Constant if @a idx is in the range of the array. Otherwise + linear in `idx - size()`. + + @liveexample{The example below shows how array elements can be read and + written using `[]` operator. Note the addition of `null` + values.,operatorarray__size_type} + + @since version 1.0.0 + */ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (is_array()) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { + m_value.array->insert(m_value.array->end(), + idx - m_value.array->size() + 1, + basic_json()); + } + + return m_value.array->operator[](idx); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified array element + + Returns a const reference to the element at specified location @a idx. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array; example: `"cannot use + operator[] with null"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + the `[]` operator.,operatorarray__size_type_const} + + @since version 1.0.0 + */ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (is_array()) + { + return m_value.array->operator[](idx); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + reference operator[](T * (&key)[n]) + { + return operator[](static_cast(key)); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @note This function is required for compatibility reasons with Clang. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + const_reference operator[](T * (&key)[n]) const + { + return operator[](static_cast(key)); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + const_reference operator[](T* key) const + { + // at only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + else + { + throw std::domain_error("cannot use operator[] with " + type_name()); + } + } + + /*! + @brief access specified object element with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(key); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const typename object_t::key_type&), this function + does not throw if the given key @a key was not found. + + @note Unlike @ref operator[](const typename object_t::key_type& key), this + function does not implicitly add an element to the position defined by @a + key. This function is furthermore also applicable to const objects. + + @param[in] key key of the element to access + @param[in] default_value the value to return if @a key is not found + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + + @since version 1.0.0 + */ + template ::value + , int>::type = 0> + ValueType value(const typename object_t::key_type& key, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return *it; + } + else + { + return default_value; + } + } + else + { + throw std::domain_error("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + */ + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /*! + @brief access specified object element via JSON Pointer with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(ptr); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const json_pointer&), this function does not throw + if the given key @a key was not found. + + @param[in] ptr a JSON pointer to the element to access + @param[in] default_value the value to return if @a ptr found no value + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value_ptr} + + @sa @ref operator[](const json_pointer&) for unchecked access by reference + + @since version 2.0.2 + */ + template ::value + , int>::type = 0> + ValueType value(const json_pointer& ptr, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if pointer resolves a value, return it or use default value + try + { + return ptr.get_checked(this); + } + catch (std::out_of_range&) + { + return default_value; + } + } + else + { + throw std::domain_error("cannot use value() with " + type_name()); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const json_pointer&, ValueType) const + */ + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /*! + @brief access the first element + + Returns a reference to the first element in the container. For a JSON + container `c`, the expression `c.front()` is equivalent to `*c.begin()`. + + @return In case of a structured type (array or object), a reference to the + first element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, guarded by assertions). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value + + @liveexample{The following code shows an example for `front()`.,front} + + @sa @ref back() -- access the last element + + @since version 1.0.0 + */ + reference front() + { + return *begin(); + } + + /*! + @copydoc basic_json::front() + */ + const_reference front() const + { + return *cbegin(); + } + + /*! + @brief access the last element + + Returns a reference to the last element in the container. For a JSON + container `c`, the expression `c.back()` is equivalent to + @code {.cpp} + auto tmp = c.end(); + --tmp; + return *tmp; + @endcode + + @return In case of a structured type (array or object), a reference to the + last element is returned. In cast of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, guarded by assertions). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value. + + @liveexample{The following code shows an example for `back()`.,back} + + @sa @ref front() -- access the first element + + @since version 1.0.0 + */ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /*! + @copydoc basic_json::back() + */ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /*! + @brief remove element given an iterator + + Removes the element specified by iterator @a pos. The iterator @a pos must + be valid and dereferenceable. Thus the `end()` iterator (which is valid, + but is not dereferenceable) cannot be used as a value for @a pos. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] pos iterator to the element to remove + @return Iterator following the last removed element. If the iterator @a + pos refers to the last element, the `end()` iterator is returned. + + @tparam InteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on an iterator which does not belong to + the current JSON value; example: `"iterator does not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterator (i.e., any iterator which is not `begin()`); example: `"iterator + out of range"` + + @complexity The complexity depends on the type: + - objects: amortized constant + - arrays: linear in distance between pos and the end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType} + + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + InteratorType erase(InteratorType pos) + { + // make sure iterator fits the current value + if (this != pos.m_object) + { + throw std::domain_error("iterator does not fit current value"); + } + + InteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not pos.m_it.primitive_iterator.is_begin()) + { + throw std::out_of_range("iterator out of range"); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove elements given an iterator range + + Removes the element specified by the range `[first; last)`. The iterator + @a first does not need to be dereferenceable if `first == last`: erasing + an empty range is a no-op. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] first iterator to the beginning of the range to remove + @param[in] last iterator past the end of the range to remove + @return Iterator following the last removed element. If the iterator @a + second refers to the last element, the `end()` iterator is returned. + + @tparam InteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on iterators which does not belong to + the current JSON value; example: `"iterators do not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterators (i.e., if `first != begin()` and `last != end()`); example: + `"iterators out of range"` + + @complexity The complexity depends on the type: + - objects: `log(size()) + std::distance(first, last)` + - arrays: linear in the distance between @a first and @a last, plus linear + in the distance between @a last and end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType_IteratorType} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template ::value or + std::is_same::value + , int>::type + = 0> + InteratorType erase(InteratorType first, InteratorType last) + { + // make sure iterator fits the current value + if (this != first.m_object or this != last.m_object) + { + throw std::domain_error("iterators do not fit current value"); + } + + InteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + throw std::out_of_range("iterators out of range"); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + return result; + } + + /*! + @brief remove element from a JSON object given a key + + Removes elements from a JSON object with the key value @a key. + + @param[in] key value of the elements to remove + + @return Number of elements removed. If @a ObjectType is the default + `std::map` type, the return value will always be `0` (@a key was not + found) or `1` (@a key was found). + + @post References and iterators to the erased elements are invalidated. + Other references and iterators are not affected. + + @throw std::domain_error when called on a type other than JSON object; + example: `"cannot use erase() with null"` + + @complexity `log(size()) + count(key)` + + @liveexample{The example shows the effect of `erase()`.,erase__key_type} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (is_object()) + { + return m_value.object->erase(key); + } + else + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + /*! + @brief remove element from a JSON array given an index + + Removes element from a JSON array at the index @a idx. + + @param[in] idx index of the element to remove + + @throw std::domain_error when called on a type other than JSON array; + example: `"cannot use erase() with null"` + @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + is out of range"` + + @complexity Linear in distance between @a idx and the end of the container. + + @liveexample{The example shows the effect of `erase()`.,erase__size_type} + + @sa @ref erase(InteratorType) -- removes the element at a given position + @sa @ref erase(InteratorType, InteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + + @since version 1.0.0 + */ + void erase(const size_type idx) + { + // this erase only works for arrays + if (is_array()) + { + if (idx >= size()) + { + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + throw std::domain_error("cannot use erase() with " + type_name()); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /*! + @brief find an element in a JSON object + + Finds an element in a JSON object with key equivalent to @a key. If the + element is not found or the JSON value is not an object, end() is + returned. + + @param[in] key key value of the element to search for + + @return Iterator to an element with key equivalent to @a key. If no such + element is found, past-the-end (see end()) iterator is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `find()` is used.,find__key_type} + + @since version 1.0.0 + */ + iterator find(typename object_t::key_type key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief find an element in a JSON object + @copydoc find(typename object_t::key_type) + */ + const_iterator find(typename object_t::key_type key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief returns the number of occurrences of a key in a JSON object + + Returns the number of elements with key @a key. If ObjectType is the + default `std::map` type, the return value will always be `0` (@a key was + not found) or `1` (@a key was found). + + @param[in] key key value of the element to count + + @return Number of elements with key @a key. If the JSON value is not an + object, the return value will be `0`. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `count()` is used.,count} + + @since version 1.0.0 + */ + size_type count(typename object_t::key_type key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(key) : 0; + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /*! + @brief returns an iterator to the first element + + Returns an iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `begin()`.,begin} + + @sa @ref cbegin() -- returns a const iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /*! + @copydoc basic_json::cbegin() + */ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /*! + @brief returns a const iterator to the first element + + Returns a const iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).begin()`. + + @liveexample{The following code shows an example for `cbegin()`.,cbegin} + + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /*! + @brief returns an iterator to one past the last element + + Returns an iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `end()`.,end} + + @sa @ref cend() -- returns a const iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /*! + @copydoc basic_json::cend() + */ + const_iterator end() const noexcept + { + return cend(); + } + + /*! + @brief returns a const iterator to one past the last element + + Returns a const iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).end()`. + + @liveexample{The following code shows an example for `cend()`.,cend} + + @sa @ref end() -- returns an iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /*! + @brief returns an iterator to the reverse-beginning + + Returns an iterator to the reverse-beginning; that is, the last element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(end())`. + + @liveexample{The following code shows an example for `rbegin()`.,rbegin} + + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /*! + @copydoc basic_json::crbegin() + */ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /*! + @brief returns an iterator to the reverse-end + + Returns an iterator to the reverse-end; that is, one before the first + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(begin())`. + + @liveexample{The following code shows an example for `rend()`.,rend} + + @sa @ref crend() -- returns a const reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /*! + @copydoc basic_json::crend() + */ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /*! + @brief returns a const reverse iterator to the last element + + Returns a const iterator to the reverse-beginning; that is, the last + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rbegin()`. + + @liveexample{The following code shows an example for `crbegin()`.,crbegin} + + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /*! + @brief returns a const reverse iterator to one before the first + + Returns a const reverse iterator to the reverse-end; that is, one before + the first element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rend()`. + + @liveexample{The following code shows an example for `crend()`.,crend} + + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + private: + // forward declaration + template class iteration_proxy; + + public: + /*! + @brief wrapper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + @note The name of this function is not yet final and may change in the + future. + */ + static iteration_proxy iterator_wrapper(reference cont) + { + return iteration_proxy(cont); + } + + /*! + @copydoc iterator_wrapper(reference) + */ + static iteration_proxy iterator_wrapper(const_reference cont) + { + return iteration_proxy(cont); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /*! + @brief checks whether the container is empty + + Checks if a JSON value has no elements. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `true` + boolean | `false` + string | `false` + number | `false` + object | result of function `object_t::empty()` + array | result of function `array_t::empty()` + + @note This function does not return whether a string stored as JSON value + is empty - it returns whether the JSON container itself is empty which is + false in the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `empty()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `begin() == end()`. + + @liveexample{The following code uses `empty()` to check if a JSON + object contains any elements.,empty} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + default: + { + // all other types are nonempty + return false; + } + } + } + + /*! + @brief returns the number of elements + + Returns the number of elements in a JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` + boolean | `1` + string | `1` + number | `1` + object | result of function object_t::size() + array | result of function array_t::size() + + @note This function does not return the length of a string stored as JSON + value - it returns the number of elements in the JSON value which is 1 in + the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their size() functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `std::distance(begin(), end())`. + + @liveexample{The following code calls `size()` on the different value + types.,size} + + @sa @ref empty() -- checks whether the container is empty + @sa @ref max_size() -- returns the maximal number of elements + + @since version 1.0.0 + */ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + default: + { + // all other types have size 1 + return 1; + } + } + } + + /*! + @brief returns the maximum possible number of elements + + Returns the maximum number of elements a JSON value is able to hold due to + system or library implementation limitations, i.e. `std::distance(begin(), + end())` for the JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` (same as `size()`) + boolean | `1` (same as `size()`) + string | `1` (same as `size()`) + number | `1` (same as `size()`) + object | result of function `object_t::max_size()` + array | result of function `array_t::max_size()` + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `max_size()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of returning `b.size()` where `b` is the largest + possible JSON value. + + @liveexample{The following code calls `max_size()` on the different value + types. Note the output is implementation specific.,max_size} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /*! + @brief clears the contents + + Clears the content of a JSON value and resets it to the default value as + if @ref basic_json(value_t) would have been called: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @note Floating-point numbers are set to `0.0` which will be serialized to + `0`. The vale type remains @ref number_float_t. + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows the effect of `clear()` to different + JSON types.,clear} + + @since version 1.0.0 + */ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + default: + { + break; + } + } + } + + /*! + @brief add an object to an array + + Appends the given element @a val to the end of the JSON value. If the + function is called on a JSON null value, an empty array is created before + appending @a val. + + @param[in] val the value to add to the JSON array + + @throw std::domain_error when called on a type other than JSON array or + null; example: `"cannot use push_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON array. Note how the `null` value was silently + converted to a JSON array.,push_back} + + @since version 1.0.0 + */ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + m_value.array->push_back(std::move(val)); + // invalidate object + val.m_type = value_t::null; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + m_value.array->push_back(val); + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + Inserts the given element @a val to the JSON object. If the function is + called on a JSON null value, an empty object is created before inserting + @a val. + + @param[in] val the value to add to the JSON object + + @throw std::domain_error when called on a type other than JSON object or + null; example: `"cannot use push_back() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON object. Note how the `null` value was silently + converted to a JSON object.,push_back__object_t__value} + + @since version 1.0.0 + */ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (not(is_null() or is_object())) + { + throw std::domain_error("cannot use push_back() with " + type_name()); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array + m_value.object->insert(val); + } + + /*! + @brief add an object to an object + @copydoc push_back(const typename object_t::value_type&) + */ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + This function allows to use `push_back` with an initializer list. In case + + 1. the current value is an object, + 2. the initializer list @a init contains only two elements, and + 3. the first element of @a init is a string, + + @a init is converted into an object element and added using + @ref push_back(const typename object_t::value_type&). Otherwise, @a init + is converted to a JSON value and added using @ref push_back(basic_json&&). + + @param init an initializer list + + @complexity Linear in the size of the initializer list @a init. + + @note This function is required to resolve an ambiguous overload error, + because pairs like `{"key", "value"}` can be both interpreted as + `object_t::value_type` or `std::initializer_list`, see + https://github.com/nlohmann/json/issues/235 for more information. + + @liveexample{The example shows how initializer lists are treated as + objects when possible.,push_back__initializer_list} + */ + void push_back(std::initializer_list init) + { + if (is_object() and init.size() == 2 and init.begin()->is_string()) + { + const string_t key = *init.begin(); + push_back(typename object_t::value_type(key, *(init.begin() + 1))); + } + else + { + push_back(basic_json(init)); + } + } + + /*! + @brief add an object to an object + @copydoc push_back(std::initializer_list) + */ + reference operator+=(std::initializer_list init) + { + push_back(init); + return *this; + } + + /*! + @brief inserts element + + Inserts element @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] val element to insert + @return iterator pointing to the inserted @a val. + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Constant plus linear in the distance between pos and end of the + container. + + @liveexample{The example shows how `insert()` is used.,insert} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); + return result; + } + else + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts element + @copydoc insert(const_iterator, const basic_json&) + */ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /*! + @brief inserts elements + + Inserts @a cnt copies of @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] cnt number of copies of @a val to insert + @param[in] val element to insert + @return iterator pointing to the first element inserted, or @a pos if + `cnt==0` + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Linear in @a cnt plus linear in the distance between @a pos + and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__count} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + return result; + } + else + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)` before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + @throw std::domain_error if @a first and @a last do not belong to the same + JSON value; example: `"iterators do not fit"` + @throw std::domain_error if @a first or @a last are iterators into + container for which insert is called; example: `"passed iterators may not + belong to container"` + + @return iterator pointing to the first element inserted, or @a pos if + `first==last` + + @complexity Linear in `std::distance(first, last)` plus linear in the + distance between @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__range} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (not is_array()) + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + throw std::domain_error("iterators do not fit"); + } + + if (first.m_object == this or last.m_object == this) + { + throw std::domain_error("passed iterators may not belong to container"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert( + pos.m_it.array_iterator, + first.m_it.array_iterator, + last.m_it.array_iterator); + return result; + } + + /*! + @brief inserts elements + + Inserts elements from initializer list @a ilist before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] ilist initializer list to insert the values from + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @return iterator pointing to the first element inserted, or @a pos if + `ilist` is empty + + @complexity Linear in `ilist.size()` plus linear in the distance between + @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__ilist} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, std::initializer_list ilist) + { + // insert only works for arrays + if (not is_array()) + { + throw std::domain_error("cannot use insert() with " + type_name()); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + throw std::domain_error("iterator does not fit current value"); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); + return result; + } + + /*! + @brief exchanges the values + + Exchanges the contents of the JSON value with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other JSON value to exchange the contents with + + @complexity Constant. + + @liveexample{The example below shows how JSON values can be swapped with + `swap()`.,swap__reference} + + @since version 1.0.0 + */ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + assert_invariant(); + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON array with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other array to exchange the contents with + + @throw std::domain_error when JSON value is not an array; example: `"cannot + use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how arrays can be swapped with + `swap()`.,swap__array_t} + + @since version 1.0.0 + */ + void swap(array_t& other) + { + // swap only works for arrays + if (is_array()) + { + std::swap(*(m_value.array), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON object with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other object to exchange the contents with + + @throw std::domain_error when JSON value is not an object; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how objects can be swapped with + `swap()`.,swap__object_t} + + @since version 1.0.0 + */ + void swap(object_t& other) + { + // swap only works for objects + if (is_object()) + { + std::swap(*(m_value.object), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON string with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other string to exchange the contents with + + @throw std::domain_error when JSON value is not a string; example: `"cannot + use swap() with boolean"` + + @complexity Constant. + + @liveexample{The example below shows how strings can be swapped with + `swap()`.,swap__string_t} + + @since version 1.0.0 + */ + void swap(string_t& other) + { + // swap only works for strings + if (is_string()) + { + std::swap(*(m_value.string), other); + } + else + { + throw std::domain_error("cannot use swap() with " + type_name()); + } + } + + /// @} + + + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + private: + /*! + @brief comparison operator for JSON types + + Returns an ordering that is similar to Python: + - order: null < boolean < number < object < array < string + - furthermore, each type is not smaller than itself + + @since version 1.0.0 + */ + friend bool operator<(const value_t lhs, const value_t rhs) noexcept + { + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) + { + return false; + } + + return order[static_cast(lhs)] < order[static_cast(rhs)]; + } + + public: + /*! + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. + - Two JSON null values are equal. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__equal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator==(const_reference v, std::nullptr_t) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, std::nullptr_t) + */ + friend bool operator==(std::nullptr_t, const_reference v) noexcept + { + return v.is_null(); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs == rhs); + } + + /*! + @brief comparison: not equal + + The functions compares the given JSON value against a null pointer. As the + null pointer can be used to initialize a JSON value to null, a comparison + of JSON value @a v with a null pointer should be equivalent to call + `not v.is_null()`. + + @param[in] v JSON value to consider + @return whether @a v is not null + + @complexity Constant. + + @liveexample{The example compares several JSON types to the null pointer. + ,operator__notequal__nullptr_t} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference v, std::nullptr_t) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, std::nullptr_t) + */ + friend bool operator!=(std::nullptr_t, const_reference v) noexcept + { + return not v.is_null(); + } + + /*! + @brief comparison: less than + + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. + - Integer and floating-point numbers are automatically converted before + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__less} + + @since version 1.0.0 + */ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array < *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object < *rhs.m_value.object; + } + case value_t::null: + { + return false; + } + case value_t::string: + { + return *lhs.m_value.string < *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean < rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer < rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float < rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /*! + @brief comparison: less than or equal + + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} + + @since version 1.0.0 + */ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return not (rhs < lhs); + } + + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__lessequal} + + @since version 1.0.0 + */ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs <= rhs); + } + + /*! + @brief comparison: greater than or equal + + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} + + @since version 1.0.0 + */ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs < rhs); + } + + /// @} + + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ + + /*! + @brief serialize to stream + + Serialize the given JSON value @a j to the output stream @a o. The JSON + value will be serialized using the @ref dump member function. The + indentation of the output can be controlled with the member variable + `width` of the output stream @a o. For instance, using the manipulator + `std::setw(4)` on @a o sets the indentation level to `4` and the + serialization result is the same as calling `dump(4)`. + + @note During serializaion, the locale and the precision of the output + stream @a o are changed. The original values are restored when the + function returns. + + @param[in,out] o stream to serialize to + @param[in] j JSON value to serialize + + @return the stream @a o + + @complexity Linear. + + @liveexample{The example below shows the serialization with different + parameters to `width` to adjust the indentation level.,operator_serialize} + + @since version 1.0.0 + */ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = (o.width() > 0); + const auto indentation = (pretty_print ? o.width() : 0); + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // fix locale problems + const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator)); + // set precision + + // 6, 15 or 16 digits of precision allows round-trip IEEE 754 + // string->float->string, string->double->string or string->long + // double->string; to be safe, we read this value from + // std::numeric_limits::digits10 + const auto old_precision = o.precision(std::numeric_limits::digits10); + + // do the actual serialization + j.dump(o, pretty_print, static_cast(indentation)); + + // reset locale and precision + o.imbue(old_locale); + o.precision(old_precision); + return o; + } + + /*! + @brief serialize to stream + @copydoc operator<<(std::ostream&, const basic_json&) + */ + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } + + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /*! + @brief deserialize from string + + @param[in] s string to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 + */ + static basic_json parse(const string_t& s, + const parser_callback_t cb = nullptr) + { + return parser(s, cb).parse(); + } + + /*! + @brief deserialize from stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__istream__parser_callback_t} + + @sa @ref parse(const string_t&, const parser_callback_t) for a version + that reads from a string + + @since version 1.0.0 + */ + static basic_json parse(std::istream& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @copydoc parse(std::istream&, const parser_callback_t) + */ + static basic_json parse(std::istream&& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @brief deserialize from stream + + Deserializes an input stream to a JSON value. + + @param[in,out] i input stream to read a serialized JSON value from + @param[in,out] j JSON value to write the deserialized input to + + @throw std::invalid_argument in case of parse errors + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below shows how a JSON value is constructed by + reading a serialization from a stream.,operator_deserialize} + + @sa parse(std::istream&, const parser_callback_t) for a variant with a + parser callback function to filter values while parsing + + @since version 1.0.0 + */ + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; + } + + /*! + @brief deserialize from stream + @copydoc operator<<(basic_json&, std::istream&) + */ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + j = parser(i).parse(); + return i; + } + + /// @} + + + private: + /////////////////////////// + // convenience functions // + /////////////////////////// + + /*! + @brief return the type as string + + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. + + @return basically a string representation of a the @ref m_type member + + @complexity Constant. + + @since version 1.0.0 + */ + std::string type_name() const + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; + } + } + + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + else + { + return res; + } + } + } + }); + } + + /*! + @brief escape a string + + Escape a string by replacing certain special characters by a sequence of + an escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. + + @param[in] s the string to escape + @return the escaped string + + @complexity Linear in the length of string @a s. + */ + static string_t escape_string(const string_t& s) + { + const auto space = extra_space(s); + if (space == 0) + { + return s; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + } + else + { + // all other characters are added as-is + result[pos++] = c; + } + break; + } + } + } + + return result; + } + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. Note that + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[out] o stream to write to + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(std::ostream& o, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) const + { + // variable to hold indentation for recursive calls + unsigned int new_indent = current_indent; + + switch (m_type) + { + case value_t::object: + { + if (m_value.object->empty()) + { + o << "{}"; + return; + } + + o << "{"; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) + { + if (i != m_value.object->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' ') << "\"" + << escape_string(i->first) << "\":" + << (pretty_print ? " " : ""); + i->second.dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') + "}"; + return; + } + + case value_t::array: + { + if (m_value.array->empty()) + { + o << "[]"; + return; + } + + o << "["; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) + { + if (i != m_value.array->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' '); + i->dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') << "]"; + return; + } + + case value_t::string: + { + o << string_t("\"") << escape_string(*m_value.string) << "\""; + return; + } + + case value_t::boolean: + { + o << (m_value.boolean ? "true" : "false"); + return; + } + + case value_t::number_integer: + { + o << m_value.number_integer; + return; + } + + case value_t::number_unsigned: + { + o << m_value.number_unsigned; + return; + } + + case value_t::number_float: + { + if (m_value.number_float == 0) + { + // special case for zero to get "0.0"/"-0.0" + o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); + } + else + { + o << m_value.number_float; + } + return; + } + + case value_t::discarded: + { + o << ""; + return; + } + + case value_t::null: + { + o << "null"; + return; + } + } + } + + private: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + + + private: + /////////////// + // iterators // + /////////////// + + /*! + @brief an iterator for primitive JSON types + + This class models an iterator for primitive JSON types (boolean, number, + string). It's only purpose is to allow the iterator/const_iterator classes + to "iterate" over primitive values. Internally, the iterator is modeled by + a `difference_type` variable. Value begin_value (`0`) models the begin, + end_value (`1`) models past the end. + */ + class primitive_iterator_t + { + public: + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return (m_it == begin_value); + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return (m_it == end_value); + } + + /// return reference to the value to change and compare + operator difference_type& () noexcept + { + return m_it; + } + + /// return value to compare + constexpr operator difference_type () const noexcept + { + return m_it; + } + + private: + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = std::numeric_limits::denorm_min(); + }; + + /*! + @brief an iterator value + + @note This structure could easily be a union, but MSVC currently does not + allow unions members with complex constructors, see + https://github.com/nlohmann/json/pull/105. + */ + struct internal_iterator + { + /// iterator for JSON objects + typename object_t::iterator object_iterator; + /// iterator for JSON arrays + typename array_t::iterator array_iterator; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator; + + /// create an uninitialized internal_iterator + internal_iterator() noexcept + : object_iterator(), array_iterator(), primitive_iterator() + {} + }; + + /// proxy class for the iterator_wrapper functions + template + class iteration_proxy + { + private: + /// helper class for iteration + class iteration_proxy_internal + { + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + size_t array_index = 0; + + public: + explicit iteration_proxy_internal(IteratorType it) noexcept + : anchor(it) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_internal& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_internal& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// inequality operator (needed for range-based for) + bool operator!= (const iteration_proxy_internal& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + typename basic_json::string_t key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + return std::to_string(array_index); + } + + // use key from the object + case value_t::object: + { + return anchor.key(); + } + + // use an empty key for all primitive types + default: + { + return ""; + } + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } + }; + + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) + : container(cont) + {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_internal begin() noexcept + { + return iteration_proxy_internal(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_internal end() noexcept + { + return iteration_proxy_internal(container.end()); + } + }; + + public: + /*! + @brief a const random access iterator for the @ref basic_json class + + This class implements a const iterator for the @ref basic_json class. From + this class, the @ref iterator class is derived. + + @note An iterator is called *initialized* when a pointer to a JSON value + has been set (e.g., by a constructor or a copy assignment). If the + iterator is default-constructed, it is *uninitialized* and most + methods are undefined. The library uses assertions to detect calls + on uninitialized iterators. + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + + @since version 1.0.0 + */ + class const_iterator : public std::iterator + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /// the type of the values when the iterator is dereferenced + using value_type = typename basic_json::value_type; + /// a type to represent differences between iterators + using difference_type = typename basic_json::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename basic_json::const_pointer; + /// defines a reference to the type iterated over (value_type) + using reference = typename basic_json::const_reference; + /// the category of the iterator + using iterator_category = std::bidirectional_iterator_tag; + + /// default constructor + const_iterator() = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit const_iterator(pointer object) noexcept + : m_object(object) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @brief copy constructor given a non-const iterator + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + explicit const_iterator(const iterator& other) noexcept + : m_object(other.m_object) + { + if (m_object != nullptr) + { + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = other.m_it.object_iterator; + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = other.m_it.array_iterator; + break; + } + + default: + { + m_it.primitive_iterator = other.m_it.primitive_iterator; + break; + } + } + } + } + + /*! + @brief copy constructor + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator(const const_iterator& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief copy assignment + @param[in,out] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + const_iterator& operator=(const_iterator other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_object, other.m_object); + std::swap(m_it, other.m_it); + return *this; + } + + private: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case basic_json::value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case basic_json::value_t::null: + { + throw std::out_of_range("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return *m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator++() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator--(int) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator--() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator==(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + throw std::domain_error("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator == other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator!=(const const_iterator& other) const + { + return not operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const const_iterator& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + throw std::domain_error("cannot compare iterators of different containers"); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot compare order of object iterators"); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator < other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const const_iterator& other) const + { + return not other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const const_iterator& other) const + { + return not operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const const_iterator& other) const + { + return not operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator+=(difference_type i) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const_iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const const_iterator& other) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use offsets with object iterators"); + } + + case basic_json::value_t::array: + { + return m_it.array_iterator - other.m_it.array_iterator; + } + + default: + { + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + throw std::domain_error("cannot use operator[] for object iterators"); + } + + case basic_json::value_t::array: + { + return *std::next(m_it.array_iterator, n); + } + + case basic_json::value_t::null: + { + throw std::out_of_range("cannot get value"); + } + + default: + { + if (m_it.primitive_iterator == -n) + { + return *m_object; + } + else + { + throw std::out_of_range("cannot get value"); + } + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + typename object_t::key_type key() const + { + assert(m_object != nullptr); + + if (m_object->is_object()) + { + return m_it.object_iterator->first; + } + else + { + throw std::domain_error("cannot use key() for non-object iterators"); + } + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + private: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator m_it = internal_iterator(); + }; + + /*! + @brief a mutable random access iterator for the @ref basic_json class + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element. + + @since version 1.0.0 + */ + class iterator : public const_iterator + { + public: + using base_iterator = const_iterator; + using pointer = typename basic_json::pointer; + using reference = typename basic_json::reference; + + /// default constructor + iterator() = default; + + /// constructor for a given JSON instance + explicit iterator(pointer object) noexcept + : base_iterator(object) + {} + + /// copy constructor + iterator(const iterator& other) noexcept + : base_iterator(other) + {} + + /// copy assignment + iterator& operator=(iterator other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + base_iterator::operator=(other); + return *this; + } + + /// return a reference to the value pointed to by the iterator + reference operator*() const + { + return const_cast(base_iterator::operator*()); + } + + /// dereference the iterator + pointer operator->() const + { + return const_cast(base_iterator::operator->()); + } + + /// post-increment (it++) + iterator operator++(int) + { + iterator result = *this; + base_iterator::operator++(); + return result; + } + + /// pre-increment (++it) + iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + iterator operator--(int) + { + iterator result = *this; + base_iterator::operator--(); + return result; + } + + /// pre-decrement (--it) + iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// subtract from iterator + iterator& operator-=(difference_type i) + { + base_iterator::operator-=(i); + return *this; + } + + /// add to iterator + iterator operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + iterator operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const iterator& other) const + { + return base_iterator::operator-(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return const_cast(base_iterator::operator[](n)); + } + + /// return the value of an iterator + reference value() const + { + return const_cast(base_iterator::value()); + } + }; + + /*! + @brief a template for a reverse iterator class + + @tparam Base the base iterator type to reverse. Valid types are @ref + iterator (to create @ref reverse_iterator) and @ref const_iterator (to + create @ref const_reverse_iterator). + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + + @since version 1.0.0 + */ + template + class json_reverse_iterator : public std::reverse_iterator + { + public: + /// shortcut to the reverse iterator adaptor + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) + {} + + /// create reverse iterator from base class + json_reverse_iterator(const base_iterator& it) noexcept + : base_iterator(it) + {} + + /// post-increment (it++) + json_reverse_iterator operator++(int) + { + return base_iterator::operator++(1); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + json_reverse_iterator operator--(int) + { + return base_iterator::operator--(1); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return this->base() - other.base(); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + typename object_t::key_type key() const + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } + }; + + + private: + ////////////////////// + // lexer and parser // + ////////////////////// + + /*! + @brief lexical analysis + + This class organizes the lexical analysis during JSON deserialization. The + core of it is a scanner generated by [re2c](http://re2c.org) that + processes a buffer and recognizes tokens according to RFC 7159. + */ + class lexer + { + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_number, ///< a number -- use get_number() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input ///< indicating the end of the input buffer + }; + + /// the char type to use in the lexer + using lexer_char_t = unsigned char; + + /// constructor with a given buffer + explicit lexer(const string_t& s) noexcept + : m_stream(nullptr), m_buffer(s) + { + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + s.size(); + } + + /// constructor with a given stream + explicit lexer(std::istream* s) noexcept + : m_stream(s), m_buffer() + { + assert(m_stream != nullptr); + std::getline(*m_stream, m_buffer); + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + m_buffer.size(); + } + + /// default constructor + lexer() = default; + + // switch off unwanted functions + lexer(const lexer&) = delete; + lexer operator=(const lexer&) = delete; + + /*! + @brief create a string from one or two Unicode code points + + There are two cases: (1) @a codepoint1 is in the Basic Multilingual + Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) + @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to + represent a code point above U+FFFF. + + @param[in] codepoint1 the code point (can be high surrogate) + @param[in] codepoint2 the code point (can be low surrogate or 0) + + @return string representation of the code point; the length of the + result string is between 1 and 4 characters. + + @throw std::out_of_range if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` + @throw std::invalid_argument if the low surrogate is invalid; example: + `""missing or wrong low surrogate""` + + @complexity Constant. + + @see + */ + static string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) + { + // calculate the code point from the given code points + std::size_t codepoint = codepoint1; + + // check if codepoint1 is a high surrogate + if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) + { + // check if codepoint2 is a low surrogate + if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) + { + codepoint = + // high surrogate occupies the most significant 22 bits + (codepoint1 << 10) + // low surrogate occupies the least significant 15 bits + + codepoint2 + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00; + } + else + { + throw std::invalid_argument("missing or wrong low surrogate"); + } + } + + string_t result; + + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + result.append(1, static_cast(codepoint)); + } + else if (codepoint <= 0x7ff) + { + // 2-byte characters: 110xxxxx 10xxxxxx + result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xffff) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0x10ffff) + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else + { + throw std::out_of_range("code points above 0x10FFFF are invalid"); + } + + return result; + } + + /// return name of values of type token_type (only used for errors) + static std::string token_type_name(const token_type t) + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_number: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + default: + { + // catch non-enum values + return "unknown token"; // LCOV_EXCL_LINE + } + } + } + + /*! + This function implements a scanner for JSON. It is specified using + regular expressions that try to follow RFC 7159 as close as possible. + These regular expressions are then translated into a minimized + deterministic finite automaton (DFA) by the tool + [re2c](http://re2c.org). As a result, the translated code for this + function consists of a large block of code with `goto` jumps. + + @return the class of the next token read from the buffer + + @complexity Linear in the length of the input.\n + + Proposition: The loop below will always terminate for finite input.\n + + Proof (by contradiction): Assume a finite input. To loop forever, the + loop must never hit code with a `break` statement. The only code + snippets without a `break` statement are the continue statements for + whitespace and byte-order-marks. To loop forever, the input must be an + infinite sequence of whitespace or byte-order-marks. This contradicts + the assumption of finite input, q.e.d. + */ + token_type scan() noexcept + { + while (true) + { + // pointer for backtracking information + m_marker = nullptr; + + // remember the begin of the token + m_start = m_cursor; + assert(m_start != nullptr); + + + { + lexer_char_t yych; + unsigned int yyaccept = 0; + static const unsigned char yybm[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 32, 0, 0, 32, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + }; + if ((m_limit - m_cursor) < 5) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + if (yych <= '\\') + { + if (yych <= '-') + { + if (yych <= '"') + { + if (yych <= 0x00) + { + goto basic_json_parser_2; + } + if (yych <= '!') + { + goto basic_json_parser_4; + } + goto basic_json_parser_9; + } + else + { + if (yych <= '+') + { + goto basic_json_parser_4; + } + if (yych <= ',') + { + goto basic_json_parser_10; + } + goto basic_json_parser_12; + } + } + else + { + if (yych <= '9') + { + if (yych <= '/') + { + goto basic_json_parser_4; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + goto basic_json_parser_15; + } + else + { + if (yych <= ':') + { + goto basic_json_parser_17; + } + if (yych == '[') + { + goto basic_json_parser_19; + } + goto basic_json_parser_4; + } + } + } + else + { + if (yych <= 't') + { + if (yych <= 'f') + { + if (yych <= ']') + { + goto basic_json_parser_21; + } + if (yych <= 'e') + { + goto basic_json_parser_4; + } + goto basic_json_parser_23; + } + else + { + if (yych == 'n') + { + goto basic_json_parser_24; + } + if (yych <= 's') + { + goto basic_json_parser_4; + } + goto basic_json_parser_25; + } + } + else + { + if (yych <= '|') + { + if (yych == '{') + { + goto basic_json_parser_26; + } + goto basic_json_parser_4; + } + else + { + if (yych <= '}') + { + goto basic_json_parser_28; + } + if (yych == 0xEF) + { + goto basic_json_parser_30; + } + goto basic_json_parser_4; + } + } + } +basic_json_parser_2: + ++m_cursor; + { + last_token_type = token_type::end_of_input; + break; + } +basic_json_parser_4: + ++m_cursor; +basic_json_parser_5: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_6: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + { + continue; + } +basic_json_parser_9: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych <= 0x1F) + { + goto basic_json_parser_5; + } + goto basic_json_parser_32; +basic_json_parser_10: + ++m_cursor; + { + last_token_type = token_type::value_separator; + break; + } +basic_json_parser_12: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_5; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + if (yych <= '9') + { + goto basic_json_parser_15; + } + goto basic_json_parser_5; +basic_json_parser_13: + yyaccept = 1; + yych = *(m_marker = ++m_cursor); + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_37; + } + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + } +basic_json_parser_14: + { + last_token_type = token_type::value_number; + break; + } +basic_json_parser_15: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yybm[0 + yych] & 64) + { + goto basic_json_parser_15; + } + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_37; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + goto basic_json_parser_14; + } +basic_json_parser_17: + ++m_cursor; + { + last_token_type = token_type::name_separator; + break; + } +basic_json_parser_19: + ++m_cursor; + { + last_token_type = token_type::begin_array; + break; + } +basic_json_parser_21: + ++m_cursor; + { + last_token_type = token_type::end_array; + break; + } +basic_json_parser_23: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'a') + { + goto basic_json_parser_39; + } + goto basic_json_parser_5; +basic_json_parser_24: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'u') + { + goto basic_json_parser_40; + } + goto basic_json_parser_5; +basic_json_parser_25: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'r') + { + goto basic_json_parser_41; + } + goto basic_json_parser_5; +basic_json_parser_26: + ++m_cursor; + { + last_token_type = token_type::begin_object; + break; + } +basic_json_parser_28: + ++m_cursor; + { + last_token_type = token_type::end_object; + break; + } +basic_json_parser_30: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 0xBB) + { + goto basic_json_parser_42; + } + goto basic_json_parser_5; +basic_json_parser_31: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; +basic_json_parser_32: + if (yybm[0 + yych] & 128) + { + goto basic_json_parser_31; + } + if (yych <= 0x1F) + { + goto basic_json_parser_33; + } + if (yych <= '"') + { + goto basic_json_parser_34; + } + goto basic_json_parser_36; +basic_json_parser_33: + m_cursor = m_marker; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } +basic_json_parser_34: + ++m_cursor; + { + last_token_type = token_type::value_string; + break; + } +basic_json_parser_36: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= 'e') + { + if (yych <= '/') + { + if (yych == '"') + { + goto basic_json_parser_31; + } + if (yych <= '.') + { + goto basic_json_parser_33; + } + goto basic_json_parser_31; + } + else + { + if (yych <= '\\') + { + if (yych <= '[') + { + goto basic_json_parser_33; + } + goto basic_json_parser_31; + } + else + { + if (yych == 'b') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + } + } + else + { + if (yych <= 'q') + { + if (yych <= 'f') + { + goto basic_json_parser_31; + } + if (yych == 'n') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 's') + { + if (yych <= 'r') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 't') + { + goto basic_json_parser_31; + } + if (yych <= 'u') + { + goto basic_json_parser_43; + } + goto basic_json_parser_33; + } + } + } +basic_json_parser_37: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_44; + } + goto basic_json_parser_33; +basic_json_parser_38: + yych = *++m_cursor; + if (yych <= ',') + { + if (yych == '+') + { + goto basic_json_parser_46; + } + goto basic_json_parser_33; + } + else + { + if (yych <= '-') + { + goto basic_json_parser_46; + } + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_47; + } + goto basic_json_parser_33; + } +basic_json_parser_39: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_49; + } + goto basic_json_parser_33; +basic_json_parser_40: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_50; + } + goto basic_json_parser_33; +basic_json_parser_41: + yych = *++m_cursor; + if (yych == 'u') + { + goto basic_json_parser_51; + } + goto basic_json_parser_33; +basic_json_parser_42: + yych = *++m_cursor; + if (yych == 0xBF) + { + goto basic_json_parser_52; + } + goto basic_json_parser_33; +basic_json_parser_43: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_54; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_54; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_54; + } + goto basic_json_parser_33; + } +basic_json_parser_44: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= 'D') + { + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_44; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_38; + } + if (yych == 'e') + { + goto basic_json_parser_38; + } + goto basic_json_parser_14; + } +basic_json_parser_46: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych >= ':') + { + goto basic_json_parser_33; + } +basic_json_parser_47: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_14; + } + if (yych <= '9') + { + goto basic_json_parser_47; + } + goto basic_json_parser_14; +basic_json_parser_49: + yych = *++m_cursor; + if (yych == 's') + { + goto basic_json_parser_55; + } + goto basic_json_parser_33; +basic_json_parser_50: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_56; + } + goto basic_json_parser_33; +basic_json_parser_51: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_58; + } + goto basic_json_parser_33; +basic_json_parser_52: + ++m_cursor; + { + continue; + } +basic_json_parser_54: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_60; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_60; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_60; + } + goto basic_json_parser_33; + } +basic_json_parser_55: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_61; + } + goto basic_json_parser_33; +basic_json_parser_56: + ++m_cursor; + { + last_token_type = token_type::literal_null; + break; + } +basic_json_parser_58: + ++m_cursor; + { + last_token_type = token_type::literal_true; + break; + } +basic_json_parser_60: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_63; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_63; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_63; + } + goto basic_json_parser_33; + } +basic_json_parser_61: + ++m_cursor; + { + last_token_type = token_type::literal_false; + break; + } +basic_json_parser_63: + ++m_cursor; + if (m_limit <= m_cursor) + { + yyfill(); // LCOV_EXCL_LINE; + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_33; + } + if (yych <= '9') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_31; + } + if (yych <= '`') + { + goto basic_json_parser_33; + } + if (yych <= 'f') + { + goto basic_json_parser_31; + } + goto basic_json_parser_33; + } + } + + } + + return last_token_type; + } + + /// append data from the stream to the internal buffer + void yyfill() noexcept + { + if (m_stream == nullptr or not * m_stream) + { + return; + } + + const auto offset_start = m_start - m_content; + const auto offset_marker = m_marker - m_start; + const auto offset_cursor = m_cursor - m_start; + + m_buffer.erase(0, static_cast(offset_start)); + std::string line; + assert(m_stream != nullptr); + std::getline(*m_stream, line); + m_buffer += "\n" + line; // add line with newline symbol + + m_content = reinterpret_cast(m_buffer.c_str()); + assert(m_content != nullptr); + m_start = m_content; + m_marker = m_start + offset_marker; + m_cursor = m_start + offset_cursor; + m_limit = m_start + m_buffer.size() - 1; + } + + /// return string representation of last read token + string_t get_token_string() const + { + assert(m_start != nullptr); + return string_t(reinterpret_cast(m_start), + static_cast(m_cursor - m_start)); + } + + /*! + @brief return string value for string tokens + + The function iterates the characters between the opening and closing + quotes of the string value. The complete string is the range + [m_start,m_cursor). Consequently, we iterate from m_start+1 to + m_cursor-1. + + We differentiate two cases: + + 1. Escaped characters. In this case, a new character is constructed + according to the nature of the escape. Some escapes create new + characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied + as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape + `"\\uxxxx"` need special care. In this case, to_unicode takes care + of the construction of the values. + 2. Unescaped characters are copied as is. + + @pre `m_cursor - m_start >= 2`, meaning the length of the last token + is at least 2 bytes which is trivially true for any string (which + consists of at least two quotes). + + " c1 c2 c3 ... " + ^ ^ + m_start m_cursor + + @complexity Linear in the length of the string.\n + + Lemma: The loop body will always terminate.\n + + Proof (by contradiction): Assume the loop body does not terminate. As + the loop body does not contain another loop, one of the called + functions must never return. The called functions are `std::strtoul` + and to_unicode. Neither function can loop forever, so the loop body + will never loop forever which contradicts the assumption that the loop + body does not terminate, q.e.d.\n + + Lemma: The loop condition for the for loop is eventually false.\n + + Proof (by contradiction): Assume the loop does not terminate. Due to + the above lemma, this can only be due to a tautological loop + condition; that is, the loop condition i < m_cursor - 1 must always be + true. Let x be the change of i for any loop iteration. Then + m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This + can be rephrased to m_cursor - m_start - 2 > x. With the + precondition, we x <= 0, meaning that the loop condition holds + indefinitly if i is always decreased. However, observe that the value + of i is strictly increasing with each iteration, as it is incremented + by 1 in the iteration expression and never decremented inside the loop + body. Hence, the loop condition will eventually be false which + contradicts the assumption that the loop condition is a tautology, + q.e.d. + + @return string value of current token without opening and closing + quotes + @throw std::out_of_range if to_unicode fails + */ + string_t get_string() const + { + assert(m_cursor - m_start >= 2); + + string_t result; + result.reserve(static_cast(m_cursor - m_start - 2)); + + // iterate the result between the quotes + for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) + { + // process escaped characters + if (*i == '\\') + { + // read next character + ++i; + + switch (*i) + { + // the default escapes + case 't': + { + result += "\t"; + break; + } + case 'b': + { + result += "\b"; + break; + } + case 'f': + { + result += "\f"; + break; + } + case 'n': + { + result += "\n"; + break; + } + case 'r': + { + result += "\r"; + break; + } + case '\\': + { + result += "\\"; + break; + } + case '/': + { + result += "/"; + break; + } + case '"': + { + result += "\""; + break; + } + + // unicode + case 'u': + { + // get code xxxx from uxxxx + auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), + 4).c_str(), nullptr, 16); + + // check if codepoint is a high surrogate + if (codepoint >= 0xD800 and codepoint <= 0xDBFF) + { + // make sure there is a subsequent unicode + if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') + { + throw std::invalid_argument("missing low surrogate"); + } + + // get code yyyy from uxxxx\uyyyy + auto codepoint2 = std::strtoul(std::string(reinterpret_cast + (i + 7), 4).c_str(), nullptr, 16); + result += to_unicode(codepoint, codepoint2); + // skip the next 10 characters (xxxx\uyyyy) + i += 10; + } + else + { + // add unicode character(s) + result += to_unicode(codepoint); + // skip the next four characters (xxxx) + i += 4; + } + break; + } + } + } + else + { + // all other characters are just copied to the end of the + // string + result.append(1, static_cast(*i)); + } + } + + return result; + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + long double str_to_float_t(long double* /* type */, char** endptr) const + { + return std::strtold(reinterpret_cast(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + double str_to_float_t(double* /* type */, char** endptr) const + { + return std::strtod(reinterpret_cast(m_start), endptr); + } + + /*! + @brief parse floating point number + + This function (and its overloads) serves to select the most approprate + standard floating point number parsing function based on the type + supplied via the first parameter. Set this to @a + static_cast(nullptr). + + @param[in] type the @ref number_float_t in use + + @param[in,out] endptr recieves a pointer to the first character after + the number + + @return the floating point number + */ + float str_to_float_t(float* /* type */, char** endptr) const + { + return std::strtof(reinterpret_cast(m_start), endptr); + } + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + This function parses the integer component up to the radix point or + exponent while collecting information about the 'floating point + representation', which it stores in the result parameter. If there is + no radix point or exponent, and the number can fit into a @ref + number_integer_t or @ref number_unsigned_t then it sets the result + parameter accordingly. + + If the number is a floating point number the number is then parsed + using @a std:strtod (or @a std:strtof or @a std::strtold). + + @param[out] result @ref basic_json object to receive the number, or + NAN if the conversion read past the current token. The latter case + needs to be treated by the caller function. + */ + void get_number(basic_json& result) const + { + assert(m_start != nullptr); + + const lexer::lexer_char_t* curptr = m_start; + + // accumulate the integer conversion result (unsigned for now) + number_unsigned_t value = 0; + + // maximum absolute value of the relevant integer type + number_unsigned_t max; + + // temporarily store the type to avoid unecessary bitfield access + value_t type; + + // look for sign + if (*curptr == '-') + { + type = value_t::number_integer; + max = static_cast((std::numeric_limits::max)()) + 1; + curptr++; + } + else + { + type = value_t::number_unsigned; + max = static_cast((std::numeric_limits::max)()); + } + + // count the significant figures + for (; curptr < m_cursor; curptr++) + { + // quickly skip tests if a digit + if (*curptr < '0' || *curptr > '9') + { + if (*curptr == '.') + { + // don't count '.' but change to float + type = value_t::number_float; + continue; + } + // assume exponent (if not then will fail parse): change to + // float, stop counting and record exponent details + type = value_t::number_float; + break; + } + + // skip if definitely not an integer + if (type != value_t::number_float) + { + // multiply last value by ten and add the new digit + auto temp = value * 10 + *curptr - '0'; + + // test for overflow + if (temp < value || temp > max) + { + // overflow + type = value_t::number_float; + } + else + { + // no overflow - save it + value = temp; + } + } + } + + // save the value (if not a float) + if (type == value_t::number_unsigned) + { + result.m_value.number_unsigned = value; + } + else if (type == value_t::number_integer) + { + result.m_value.number_integer = -static_cast(value); + } + else + { + // parse with strtod + result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + } + + // save the type + result.m_type = type; + } + + private: + /// optional input stream + std::istream* m_stream = nullptr; + /// the buffer + string_t m_buffer; + /// the buffer pointer + const lexer_char_t* m_content = nullptr; + /// pointer to the beginning of the current symbol + const lexer_char_t* m_start = nullptr; + /// pointer for backtracking information + const lexer_char_t* m_marker = nullptr; + /// pointer to the current symbol + const lexer_char_t* m_cursor = nullptr; + /// pointer to the end of the buffer + const lexer_char_t* m_limit = nullptr; + /// the last token type + token_type last_token_type = token_type::end_of_input; + }; + + /*! + @brief syntax analysis + + This class implements a recursive decent parser. + */ + class parser + { + public: + /// constructor for strings + parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(s) + { + // read first token + get_token(); + } + + /// a parser reading from an input stream + parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept + : callback(cb), m_lexer(&_is) + { + // read first token + get_token(); + } + + /// public parser interface + basic_json parse() + { + basic_json result = parse_internal(true); + result.assert_invariant(); + + expect(lexer::token_type::end_of_input); + + // return parser result and replace it with null in case the + // top-level value was discarded by the callback function + return result.is_discarded() ? basic_json() : std::move(result); + } + + private: + /// the actual parser + basic_json parse_internal(bool keep) + { + auto result = basic_json(value_t::discarded); + + switch (last_token) + { + case lexer::token_type::begin_object: + { + if (keep and (not callback or (keep = callback(depth++, parse_event_t::object_start, result)))) + { + // explicitly set result to object to cope with {} + result.m_type = value_t::object; + result.m_value = value_t::object; + } + + // read next token + get_token(); + + // closing } -> we are done + if (last_token == lexer::token_type::end_object) + { + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse key-value pairs + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // store key + expect(lexer::token_type::value_string); + const auto key = m_lexer.get_string(); + + bool keep_tag = false; + if (keep) + { + if (callback) + { + basic_json k(key); + keep_tag = callback(depth, parse_event_t::key, k); + } + else + { + keep_tag = true; + } + } + + // parse separator (:) + get_token(); + expect(lexer::token_type::name_separator); + + // parse and add value + get_token(); + auto value = parse_internal(keep); + if (keep and keep_tag and not value.is_discarded()) + { + result[key] = std::move(value); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing } + expect(lexer::token_type::end_object); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::begin_array: + { + if (keep and (not callback or (keep = callback(depth++, parse_event_t::array_start, result)))) + { + // explicitly set result to object to cope with [] + result.m_type = value_t::array; + result.m_value = value_t::array; + } + + // read next token + get_token(); + + // closing ] -> we are done + if (last_token == lexer::token_type::end_array) + { + get_token(); + if (callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse values + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // parse value + auto value = parse_internal(keep); + if (keep and not value.is_discarded()) + { + result.push_back(std::move(value)); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing ] + expect(lexer::token_type::end_array); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::literal_null: + { + get_token(); + result.m_type = value_t::null; + break; + } + + case lexer::token_type::value_string: + { + const auto s = m_lexer.get_string(); + get_token(); + result = basic_json(s); + break; + } + + case lexer::token_type::literal_true: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = true; + break; + } + + case lexer::token_type::literal_false: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = false; + break; + } + + case lexer::token_type::value_number: + { + m_lexer.get_number(result); + get_token(); + break; + } + + default: + { + // the last token was unexpected + unexpect(last_token); + } + } + + if (keep and callback and not callback(depth, parse_event_t::value, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + /// get next token from lexer + typename lexer::token_type get_token() noexcept + { + last_token = m_lexer.scan(); + return last_token; + } + + void expect(typename lexer::token_type t) const + { + if (t != last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + error_msg += "; expected " + lexer::token_type_name(t); + throw std::invalid_argument(error_msg); + } + } + + void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + throw std::invalid_argument(error_msg); + } + } + + private: + /// current level of recursion + int depth = 0; + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + typename lexer::token_type last_token = lexer::token_type::uninitialized; + /// the lexer + lexer m_lexer; + }; + + public: + /*! + @brief JSON Pointer + + A JSON pointer defines a string syntax for identifying a specific value + within a JSON document. It can be used with functions `at` and + `operator[]`. Furthermore, JSON pointers are the base for JSON patches. + + @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + class json_pointer + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /*! + @brief create JSON pointer + + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). + + @param[in] s string representing the JSON pointer; if omitted, the + empty string is assumed which references the whole JSON + value + + @throw std::domain_error if reference token is nonempty and does not + begin with a slash (`/`); example: `"JSON pointer must be empty or + begin with /"` + @throw std::domain_error if a tilde (`~`) is not followed by `0` + (representing `~`) or `1` (representing `/`); example: `"escape error: + ~ must be followed with 0 or 1"` + + @liveexample{The example shows the construction several valid JSON + pointers as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /*! + @brief return a string representation of the JSON pointer + + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode + + @return a string representation of the JSON pointer + + @liveexample{The example shows the result of `to_string`., + json_pointer__to_string} + + @since version 2.0.0 + */ + std::string to_string() const noexcept + { + return std::accumulate(reference_tokens.begin(), + reference_tokens.end(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); + } + + /// @copydoc to_string() + operator std::string() const + { + return to_string(); + } + + private: + /// remove and return last reference pointer + std::string pop_back() + { + if (is_root()) + { + throw std::domain_error("JSON pointer has no parent"); + } + + auto last = reference_tokens.back(); + reference_tokens.pop_back(); + return last; + } + + /// return whether pointer points to the root document + bool is_root() const + { + return reference_tokens.empty(); + } + + json_pointer top() const + { + if (is_root()) + { + throw std::domain_error("JSON pointer has no parent"); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + */ + reference get_and_create(reference j) const + { + pointer result = &j; + + // in case no reference tokens exist, return a reference to the + // JSON value j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case value_t::array: + { + // create an entry in the array + result = &result->operator[](static_cast(std::stoi(reference_token))); + break; + } + + /* + The following code is only reached if there exists a + reference token _and_ the current value is primitive. In + this case, we have an error situation, because primitive + values may only occur as single value; that is, with an + empty list of reference tokens. + */ + default: + { + throw std::domain_error("invalid value to unflatten"); + } + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + */ + reference get_unchecked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + if (reference_token == "-") + { + // explicityly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + reference get_checked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + */ + const_reference get_unchecked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" cannot be used for const access + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // use unchecked array access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + const_reference get_checked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + throw std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range"); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + throw std::domain_error("array index must not begin with '0'"); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + } + } + } + + return *ptr; + } + + /// split the string input to reference tokens + static std::vector split(std::string reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (reference_string[0] != '/') + { + throw std::domain_error("JSON pointer must be empty or begin with '/'"); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + size_t slash = reference_string.find_first_of("/", 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == string::npos+1 = 0 + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = slash + 1, + // find next slash + slash = reference_string.find_first_of("/", start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (size_t pos = reference_token.find_first_of("~"); + pos != std::string::npos; + pos = reference_token.find_first_of("~", pos + 1)) + { + assert(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1')) + { + throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); + } + } + + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @brief replace all occurrences of a substring by another string + + @param[in,out] s the string to manipulate + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f + + @return The string @a s where all occurrences of @a f are replaced + with @a t. + + @pre The search string @a f must not be empty. + + @since version 2.0.0 + */ + static void replace_substring(std::string& s, + const std::string& f, + const std::string& t) + { + assert(not f.empty()); + + for ( + size_t pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t + pos = s.find(f, pos + t.size()) // find next occurrence of f + ); + } + + /// escape tilde and slash + static std::string escape(std::string s) + { + // escape "~"" to "~0" and "/" to "~1" + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; + } + + /// unescape tilde and slash + static void unescape(std::string& s) + { + // first transform any occurrence of the sequence '~1' to '/' + replace_substring(s, "~1", "/"); + // then transform any occurrence of the sequence '~0' to '~' + replace_substring(s, "~0", "~"); + } + + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const basic_json& value, + basic_json& result) + { + switch (value.m_type) + { + case value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), + element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + */ + static basic_json unflatten(const basic_json& value) + { + if (not value.is_object()) + { + throw std::domain_error("only objects can be unflattened"); + } + + basic_json result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (not element.second.is_primitive()) + { + throw std::domain_error("values in object must be primitive"); + } + + // assign value to reference pointed to by JSON pointer; Note + // that if the JSON pointer is "" (i.e., points to the whole + // value), function get_and_create returns a reference to + // result itself. An assignment will then create a primitive + // value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + private: + /// the reference tokens + std::vector reference_tokens {}; + }; + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. Similar to @ref operator[](const typename + object_t::key_type&), `null` values are created in arrays and objects if + necessary. + + In particular: + - If the JSON pointer points to an object key that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. + - If the JSON pointer points to an array index that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. All indices between the current maximum and the given + index are also filled with `null`. + - The special value `-` is treated as a synonym for the index past the + end. + + @param[in] ptr a JSON pointer + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer} + + @since version 2.0.0 + */ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. The function does not change the JSON + value; no `null` values are created. In particular, the the special value + `-` yields an exception. + + @param[in] ptr JSON pointer to the desired element + + @return const reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} + + @since version 2.0.0 + */ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a reference to the element at with specified JSON pointer @a ptr, + with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer} + + @since version 2.0.0 + */ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a const reference to the element at with specified JSON pointer @a + ptr, with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} + + @since version 2.0.0 + */ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /*! + @brief return flattened JSON value + + The function creates a JSON object whose keys are JSON pointers (see [RFC + 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all + primitive. The original JSON value can be restored using the @ref + unflatten() function. + + @return an object that maps JSON pointers to primitve values + + @note Empty objects and arrays are flattened to `null` and will not be + reconstructed correctly by the @ref unflatten() function. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a JSON object is flattened to an + object whose keys consist of JSON pointers.,flatten} + + @sa @ref unflatten() for the reverse function + + @since version 2.0.0 + */ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /*! + @brief unflatten a previously flattened JSON value + + The function restores the arbitrary nesting of a JSON value that has been + flattened before using the @ref flatten() function. The JSON value must + meet certain constraints: + 1. The value must be an object. + 2. The keys must be JSON pointers (see + [RFC 6901](https://tools.ietf.org/html/rfc6901)) + 3. The mapped values must be primitive JSON types. + + @return the original JSON from a flattened version + + @note Empty objects and arrays are flattened by @ref flatten() to `null` + values and can not unflattened to their original type. Apart from + this example, for a JSON value `j`, the following is always true: + `j == j.flatten().unflatten()`. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a flattened JSON object is + unflattened into the original nested JSON object.,unflatten} + + @sa @ref flatten() for the reverse function + + @since version 2.0.0 + */ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /*! + @brief applies a JSON patch + + [JSON Patch](http://jsonpatch.com) defines a JSON document structure for + expressing a sequence of operations to apply to a JSON) document. With + this funcion, a JSON Patch is applied to the current JSON value by + executing all operations from the patch. + + @param[in] json_patch JSON patch document + @return patched document + + @note The application of a patch is atomic: Either all operations succeed + and the patched document is returned or an exception is thrown. In + any case, the original value is not changed: the patch is applied + to a copy of the value. + + @throw std::out_of_range if a JSON pointer inside the patch could not + be resolved successfully in the current JSON value; example: `"key baz + not found"` + @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + attributes are missing); example: `"operation add must have member path"` + + @complexity Linear in the size of the JSON value and the length of the + JSON patch. As usually only a fraction of the JSON value is affected by + the patch, the complexity can usually be neglected. + + @liveexample{The following code shows how a JSON patch is applied to a + value.,patch} + + @sa @ref diff -- create a JSON patch by comparing two JSON values + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.is_root()) + { + result = val; + } + else + { + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + basic_json& x = result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = std::stoi(last_path); + if (static_cast(idx) > parent.size()) + { + // avoid undefined behavior + throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + } + else + { + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + } + break; + } + + default: + { + // if there exists a parent it cannot be primitive + assert(false); // LCOV_EXCL_LINE + } + } + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [&result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (it != parent.end()) + { + parent.erase(it); + } + else + { + throw std::out_of_range("key '" + last_path + "' not found"); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(static_cast(std::stoi(last_path))); + } + }; + + // type check + if (not json_patch.is_array()) + { + // a JSON patch must be an array of objects + throw std::invalid_argument("JSON patch must be an array of objects"); + } + + // iterate and apply th eoperations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json& + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (it == val.m_value.object->end()) + { + throw std::invalid_argument(error_msg + " must have member '" + member + "'"); + } + + // check if result is of type string + if (string_type and not it->second.is_string()) + { + throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); + } + + // no error: return value + return it->second; + }; + + // type check + if (not val.is_object()) + { + throw std::invalid_argument("JSON patch must be an array of objects"); + } + + // collect mandatory members + const std::string op = get_value("op", "op", true); + const std::string path = get_value(op, "path", true); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const std::string from_path = get_value("move", "from", true); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const std::string from_path = get_value("copy", "from", true);; + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + result[ptr] = result.at(from_ptr); + break; + } + + case patch_operations::test: + { + bool success = false; + try + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + catch (std::out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (not success) + { + throw std::domain_error("unsuccessful: " + val.dump()); + } + + break; + } + + case patch_operations::invalid: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + throw std::invalid_argument("operation value '" + op + "' is invalid"); + } + } + } + + return result; + } + + /*! + @brief creates a diff as a JSON patch + + Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can + be changed into the value @a target by calling @ref patch function. + + @invariant For two JSON values @a source and @a target, the following code + yields always `true`: + @code {.cpp} + source.patch(diff(source, target)) == target; + @endcode + + @note Currently, only `remove`, `add`, and `replace` operations are + generated. + + @param[in] source JSON value to copare from + @param[in] target JSON value to copare against + @param[in] path helper value to create JSON pointers + + @return a JSON patch to convert the @a source to @a target + + @complexity Linear in the lengths of @a source and @a target. + + @liveexample{The following code shows how a JSON patch is created as a + diff for two JSON values.,diff} + + @sa @ref patch -- apply a JSON patch + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + + @since version 2.0.0 + */ + static basic_json diff(const basic_json& source, + const basic_json& target, + std::string path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + } + else + { + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + size_t i = 0; + while (i < source.size() and i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.begin(); it != source.end(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, + {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + break; + } + } + } + + return result; + } + + /// @} +}; + + +///////////// +// presets // +///////////// + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} + + +/////////////////////// +// nonmember support // +/////////////////////// + +// specialization of std::swap, and std::hash +namespace std +{ +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template <> +inline void swap(nlohmann::json& j1, + nlohmann::json& j2) noexcept( + is_nothrow_move_constructible::value and + is_nothrow_move_assignable::value + ) +{ + j1.swap(j2); +} + +/// hash value for JSON objects +template <> +struct hash +{ + /*! + @brief return a hash value for a JSON object + + @since version 1.0.0 + */ + std::size_t operator()(const nlohmann::json& j) const + { + // a naive hashing via the string representation + const auto& h = hash(); + return h(j.dump()); + } +}; +} + +/*! +@brief user-defined string literal for JSON values + +This operator implements a user-defined string literal for JSON objects. It +can be used by adding `"_json"` to a string literal and returns a JSON object +if no parse error occurred. + +@param[in] s a string representation of a JSON object +@return a JSON object + +@since version 1.0.0 +*/ +inline nlohmann::json operator "" _json(const char* s, std::size_t) +{ + return nlohmann::json::parse(reinterpret_cast(s)); +} + +/*! +@brief user-defined string literal for JSON pointer + +This operator implements a user-defined string literal for JSON Pointers. It +can be used by adding `"_json"` to a string literal and returns a JSON pointer +object if no parse error occurred. + +@param[in] s a string representation of a JSON Pointer +@return a JSON pointer object + +@since version 2.0.0 +*/ +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t) +{ + return nlohmann::json::json_pointer(s); +} + +// restore GCC/clang diagnostic settings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic pop +#endif + +#endif diff --git a/ext/offbase/README.md b/ext/offbase/README.md deleted file mode 100644 index 6668b878..00000000 --- a/ext/offbase/README.md +++ /dev/null @@ -1,59 +0,0 @@ -A super-minimal in-filesystem persistent JSON object -====== - -[We](https://www.zerotier.com/) like minimalism. - -Offbase is an extension of the excellent [nlohmann/json](https://github.com/nlohmann/json) C++11 JSON class that adds simple object persistence to/from the filesystem. Objects are stored into a directory hierarchy in fully "expanded" form with each field/value being represented by a separate file. - -Features: - - - Very easy to use - - Minimal! - - Easy to understand and maintain - - Trivial to implement in other languages - - No dependencies beyond standard libraries - - Small code footprint in both source and binary form - - Easy to port to other platforms - - Exactly reproduces JSON object hierarchies including all JSON type information - - Database can be explored from the shell, browsed in a web browser or file explorer, scanned with `find` and `grep`, etc. - - Database can be backed up, restored, versioned, etc. with tools like `git`, `rsync`, `duplicity`, etc. - - Alien files like `.git` or `.DS_Store` are harmlessly ignored if present - - Saving only changes what's changed to reduce I/O overhead and SSD wear - -Limitations and shortcomings: - - - This creates a lot of tiny files, which is inefficient on some filesystems and might run into inode limits in extreme cases. For data sets with more than, say, a million items we recommend a filesystem like `btrfs` or `reiserfs`. Things like [redisfs](https://steve.fi/Software/redisfs/) are also worth exploring. On Linux another alternative is to put the database into `/dev/shm` (RAM disk) and then regularly back it up with `duplicity` or similar. - - The whole JSON object is held in memory *twice* for diffing purposes. - - Diffing traverses the whole tree and then updates the shadow copy, which makes `commit()` slow for huge data sets. This is not suitable for "big" data where "big" here is probably more than a few hundred megabytes. - - Recursion is used, so if you have object hierarchies that are incredibly deep (hundreds or more) it might be possible to overflow your stack and crash your app. - - This is not thread safe and must be guarded by a mutex if used in a threaded app. - -Caveats: - - - Key names are escaped for safety in the filesystem, but we still don't recommend allowing external users to set just anything into your JSON store. See the point about recursion under limitations. - -Future: - - - It would not be too hard to tie this into a filesystem change monitoring API and automatically read changes from disk if they are detected. This would allow the database to be edited "live" in the filesystem. - - In theory this could provide replication or clustering through distributed filesystems, file syncing, or things like [Amazon Elastic Filesystem](https://aws.amazon.com/efs/). - - Recursion could be factored out to get rid of any object hierarchy depth constraints. - - Mutexes could be integrated somehow to allow for finer grained locking in multithreaded apps. - - Diffing and selective updates could be made more memory and CPU efficient using hashes, etc. - -## How to Use - -The `offbase` class just extends [nlohmann::json](https://github.com/nlohmann/json) and gives you a JSON object. Take care to make sure you don't change the type of the 'root' object represented by the 'offbase' instance from JSON 'object'. Anything under it can of course be any JSON type, including any object. - -Just put data into the object and then periodically call `commit()` to persist changes to disk. The `commit()` method diffs the current contents of the object with what it knows to have been previously persisted to disk and modifies the representation on disk to match. This can be done after writes or periodically in a background thread. - -See comments in `offbase.hpp` for full documentation including details about error handling, etc. - -## Persistence format - -The base object represented by the `offbase` instance is persisted into a directory hierarchy under its base path. Files and directories are named according to a simple convention of `keyname.typecode` where `keyname` is an escaped key name (or hex array index in the case of arrays) and `typecode` is a single character indicating whether the item is a JSON value, array, or object. - - - `*.V`: JSON values (actual value type is inferred during JSON parse) - - `*.O`: JSON objects (these are subdirectories) - - `*.A`: JSON arrays (also subdirectories containing items by hex array index) - -There are in theory simpler ways to represent JSON in a filesystem, such as the "flattened" JSON "pointer" format, but this has the disadvantage of not disambiguating objects vs. arrays. Offbase's persistence format is designed to perfectly reproduce the exact same JSON tree on load as was most recently committed. diff --git a/ext/offbase/json/LICENSE.MIT b/ext/offbase/json/LICENSE.MIT deleted file mode 100644 index e2ac4891..00000000 --- a/ext/offbase/json/LICENSE.MIT +++ /dev/null @@ -1,22 +0,0 @@ -The library is licensed under the MIT License -: - -Copyright (c) 2013-2016 Niels Lohmann - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/ext/offbase/json/README.md b/ext/offbase/json/README.md deleted file mode 100644 index c0bb61b1..00000000 --- a/ext/offbase/json/README.md +++ /dev/null @@ -1,511 +0,0 @@ -[![JSON for Modern C++](https://raw.githubusercontent.com/nlohmann/json/master/doc/json.gif)](https://github.com/nlohmann/json/releases) - -[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) -[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk?svg=true)](https://ci.appveyor.com/project/nlohmann/json) -[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) -[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) -[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/p5o4znPnGHJpDVqN) -[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) -[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) -[![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) -[![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) - -## Design goals - -There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: - -- **Intuitive syntax**. In languages such as Python, JSON feels like a first class data type. We used all the operator magic of modern C++ to achieve the same feeling in your code. Check out the [examples below](#examples) and you know, what I mean. - -- **Trivial integration**. Our whole code consists of a single header file [`json.hpp`](https://github.com/nlohmann/json/blob/develop/src/json.hpp). That's it. No library, no subproject, no dependencies, no complex build system. The class is written in vanilla C++11. All in all, everything should require no adjustment of your compiler flags or project settings. - -- **Serious testing**. Our class is heavily [unit-tested](https://github.com/nlohmann/json/blob/master/test/src/unit.cpp) and covers [100%](https://coveralls.io/r/nlohmann/json) of the code, including all exceptional behavior. Furthermore, we checked with [Valgrind](http://valgrind.org) that there are no memory leaks. - -Other aspects were not so important to us: - -- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. - -- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). - -See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. - - -## Integration - -The single required source, file `json.hpp` is in the `src` directory or [released here](https://github.com/nlohmann/json/releases). All you need to do is add - -```cpp -#include "json.hpp" - -// for convenience -using json = nlohmann::json; -``` - -to the files you want to use JSON objects. That's it. Do not forget to set the necessary switches to enable C++11 (e.g., `-std=c++11` for GCC and Clang). - -:beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. - - -## Examples - -Here are some examples to give you an idea how to use the class. - -Assume you want to create the JSON object - -```json -{ - "pi": 3.141, - "happy": true, - "name": "Niels", - "nothing": null, - "answer": { - "everything": 42 - }, - "list": [1, 0, 2], - "object": { - "currency": "USD", - "value": 42.99 - } -} -``` - -With the JSON class, you could write: - -```cpp -// create an empty structure (null) -json j; - -// add a number that is stored as double (note the implicit conversion of j to an object) -j["pi"] = 3.141; - -// add a Boolean that is stored as bool -j["happy"] = true; - -// add a string that is stored as std::string -j["name"] = "Niels"; - -// add another null object by passing nullptr -j["nothing"] = nullptr; - -// add an object inside the object -j["answer"]["everything"] = 42; - -// add an array that is stored as std::vector (using an initializer list) -j["list"] = { 1, 0, 2 }; - -// add another object (using an initializer list of pairs) -j["object"] = { {"currency", "USD"}, {"value", 42.99} }; - -// instead, you could also write (which looks very similar to the JSON above) -json j2 = { - {"pi", 3.141}, - {"happy", true}, - {"name", "Niels"}, - {"nothing", nullptr}, - {"answer", { - {"everything", 42} - }}, - {"list", {1, 0, 2}}, - {"object", { - {"currency", "USD"}, - {"value", 42.99} - }} -}; -``` - -Note that in all these cases, you never need to "tell" the compiler which JSON value you want to use. If you want to be explicit or express some edge cases, the functions `json::array` and `json::object` will help: - -```cpp -// a way to express the empty array [] -json empty_array_explicit = json::array(); - -// ways to express the empty object {} -json empty_object_implicit = json({}); -json empty_object_explicit = json::object(); - -// a way to express an _array_ of key/value pairs [["currency", "USD"], ["value", 42.99]] -json array_not_object = { json::array({"currency", "USD"}), json::array({"value", 42.99}) }; -``` - - -### Serialization / Deserialization - -You can create an object (deserialization) by appending `_json` to a string literal: - -```cpp -// create object from string literal -json j = "{ \"happy\": true, \"pi\": 3.141 }"_json; - -// or even nicer with a raw string literal -auto j2 = R"( - { - "happy": true, - "pi": 3.141 - } -)"_json; - -// or explicitly -auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); -``` - -You can also get a string representation (serialize): - -```cpp -// explicit conversion to string -std::string s = j.dump(); // {\"happy\":true,\"pi\":3.141} - -// serialization with pretty printing -// pass in the amount of spaces to indent -std::cout << j.dump(4) << std::endl; -// { -// "happy": true, -// "pi": 3.141 -// } -``` - -You can also use streams to serialize and deserialize: - -```cpp -// deserialize from standard input -json j; -std::cin >> j; - -// serialize to standard output -std::cout << j; - -// the setw manipulator was overloaded to set the indentation for pretty printing -std::cout << std::setw(4) << j << std::endl; -``` - -These operators work for any subclasses of `std::istream` or `std::ostream`. - -Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. - - -### STL-like access - -We designed the JSON class to behave just like an STL container. In fact, it satisfies the [**ReversibleContainer**](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) requirement. - -```cpp -// create an array using push_back -json j; -j.push_back("foo"); -j.push_back(1); -j.push_back(true); - -// iterate the array -for (json::iterator it = j.begin(); it != j.end(); ++it) { - std::cout << *it << '\n'; -} - -// range-based for -for (auto& element : j) { - std::cout << element << '\n'; -} - -// getter/setter -const std::string tmp = j[0]; -j[1] = 42; -bool foo = j.at(2); - -// other stuff -j.size(); // 3 entries -j.empty(); // false -j.type(); // json::value_t::array -j.clear(); // the array is empty again - -// convenience type checkers -j.is_null(); -j.is_boolean(); -j.is_number(); -j.is_object(); -j.is_array(); -j.is_string(); - -// comparison -j == "[\"foo\", 1, true]"_json; // true - -// create an object -json o; -o["foo"] = 23; -o["bar"] = false; -o["baz"] = 3.141; - -// special iterator member functions for objects -for (json::iterator it = o.begin(); it != o.end(); ++it) { - std::cout << it.key() << " : " << it.value() << "\n"; -} - -// find an entry -if (o.find("foo") != o.end()) { - // there is an entry with key "foo" -} - -// or simpler using count() -int foo_present = o.count("foo"); // 1 -int fob_present = o.count("fob"); // 0 - -// delete an entry -o.erase("foo"); -``` - - -### Conversion from STL containers - -Any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values can be used to construct JSON types (e.g., integers, floating point numbers, Booleans, string types, or again STL containers described in this section) can be used to create a JSON array. The same holds for similar associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`), but in these cases the order of the elements of the array depends how the elements are ordered in the respective STL container. - -```cpp -std::vector c_vector {1, 2, 3, 4}; -json j_vec(c_vector); -// [1, 2, 3, 4] - -std::deque c_deque {1.2, 2.3, 3.4, 5.6}; -json j_deque(c_deque); -// [1.2, 2.3, 3.4, 5.6] - -std::list c_list {true, true, false, true}; -json j_list(c_list); -// [true, true, false, true] - -std::forward_list c_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; -json j_flist(c_flist); -// [12345678909876, 23456789098765, 34567890987654, 45678909876543] - -std::array c_array {{1, 2, 3, 4}}; -json j_array(c_array); -// [1, 2, 3, 4] - -std::set c_set {"one", "two", "three", "four", "one"}; -json j_set(c_set); // only one entry for "one" is used -// ["four", "one", "three", "two"] - -std::unordered_set c_uset {"one", "two", "three", "four", "one"}; -json j_uset(c_uset); // only one entry for "one" is used -// maybe ["two", "three", "four", "one"] - -std::multiset c_mset {"one", "two", "one", "four"}; -json j_mset(c_mset); // only one entry for "one" is used -// maybe ["one", "two", "four"] - -std::unordered_multiset c_umset {"one", "two", "one", "four"}; -json j_umset(c_umset); // both entries for "one" are used -// maybe ["one", "two", "one", "four"] -``` - -Likewise, any associative key-value containers (`std::map`, `std::multimap`, `std::unordered_map`, `std::unordered_multimap`) whose keys can construct an `std::string` and whose values can be used to construct JSON types (see examples above) can be used to to create a JSON object. Note that in case of multimaps only one key is used in the JSON object and the value depends on the internal order of the STL container. - -```cpp -std::map c_map { {"one", 1}, {"two", 2}, {"three", 3} }; -json j_map(c_map); -// {"one": 1, "three": 3, "two": 2 } - -std::unordered_map c_umap { {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; -json j_umap(c_umap); -// {"one": 1.2, "two": 2.3, "three": 3.4} - -std::multimap c_mmap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; -json j_mmap(c_mmap); // only one entry for key "three" is used -// maybe {"one": true, "two": true, "three": true} - -std::unordered_multimap c_ummap { {"one", true}, {"two", true}, {"three", false}, {"three", true} }; -json j_ummap(c_ummap); // only one entry for key "three" is used -// maybe {"one": true, "two": true, "three": true} -``` - -### JSON Pointer and JSON Patch - -The library supports **JSON Pointer** ([RFC 6901](https://tools.ietf.org/html/rfc6901)) as alternative means to address structured values. On top of this, **JSON Patch** ([RFC 6902](https://tools.ietf.org/html/rfc6902)) allows to describe differences between two JSON values - effectively allowing patch and diff operations known from Unix. - -```cpp -// a JSON value -json j_original = R"({ - "baz": ["one", "two", "three"], - "foo": "bar" -})"_json; - -// access members with a JSON pointer (RFC 6901) -j_original["/baz/1"_json_pointer]; -// "two" - -// a JSON patch (RFC 6902) -json j_patch = R"([ - { "op": "replace", "path": "/baz", "value": "boo" }, - { "op": "add", "path": "/hello", "value": ["world"] }, - { "op": "remove", "path": "/foo"} -])"_json; - -// apply the patch -json j_result = j_original.patch(j_patch); -// { -// "baz": "boo", -// "hello": ["world"] -// } - -// calculate a JSON patch from two JSON values -json::diff(j_result, j_original); -// [ -// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, -// { "op": "remove","path": "/hello" }, -// { "op": "add", "path": "/foo", "value": "bar" } -// ] -``` - - -### Implicit conversions - -The type of the JSON object is determined automatically by the expression to store. Likewise, the stored value is implicitly converted. - -```cpp -// strings -std::string s1 = "Hello, world!"; -json js = s1; -std::string s2 = js; - -// Booleans -bool b1 = true; -json jb = b1; -bool b2 = jb; - -// numbers -int i = 42; -json jn = i; -double f = jn; - -// etc. -``` - -You can also explicitly ask for the value: - -```cpp -std::string vs = js.get(); -bool vb = jb.get(); -int vi = jn.get(); - -// etc. -``` - - -## Supported compilers - -Though it's 2016 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work: - -- GCC 4.9 - 6.0 (and possibly later) -- Clang 3.4 - 3.9 (and possibly later) -- Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) - -I would be happy to learn about other compilers/versions. - -Please note: - -- GCC 4.8 does not work because of two bugs ([55817](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55817) and [57824](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57824)) in the C++11 support. Note there is a [pull request](https://github.com/nlohmann/json/pull/212) to fix some of the issues. -- Android defaults to using very old compilers and C++ libraries. To fix this, add the following to your `Application.mk`. This will switch to the LLVM C++ library, the Clang compiler, and enable C++11 and other features disabled by default. - - ``` - APP_STL := c++_shared - NDK_TOOLCHAIN_VERSION := clang3.6 - APP_CPPFLAGS += -frtti -fexceptions - ``` - - The code compiles successfully with [Android NDK](https://developer.android.com/ndk/index.html?hl=ml), Revision 9 - 11 (and possibly later) and [CrystaX's Android NDK](https://www.crystax.net/en/android/ndk) version 10. - -- For GCC running on MinGW or Android SDK, the error `'to_string' is not a member of 'std'` (or similarly, for `strtod`) may occur. Note this is not an issue with the code, but rather with the compiler itself. On Android, see above to build with a newer environment. For MinGW, please refer to [this site](http://tehsausage.com/mingw-to-string) and [this discussion](https://github.com/nlohmann/json/issues/136) for information on how to fix this bug. For Android NDK using `APP_STL := gnustl_static`, please refer to [this discussion](https://github.com/nlohmann/json/issues/219). - -The following compilers are currently used in continuous integration at [Travis](https://travis-ci.org/nlohmann/json) and [AppVeyor](https://ci.appveyor.com/project/nlohmann/json): - -| Compiler | Operating System | Version String | -|-----------------|------------------------------|----------------| -| GCC 4.9.3 | Ubuntu 14.04.4 LTS | g++-4.9 (Ubuntu 4.9.3-8ubuntu2~14.04) 4.9.3 | -| GCC 5.3.0 | Ubuntu 14.04.4 LTS | g++-5 (Ubuntu 5.3.0-3ubuntu1~14.04) 5.3.0 20151204 | -| GCC 6.1.1 | Ubuntu 14.04.4 LTS | g++-6 (Ubuntu 6.1.1-3ubuntu11~14.04.1) 6.1.1 20160511 | -| Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | -| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | -| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | -| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | -| Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | -| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | -| Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.5.0 (OSX 10.11.5) | Apple LLVM version 8.0.0 (clang-800.0.24.1) | -| Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | - - -## License - - - -The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): - -Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -## Thanks - -I deeply appreciate the help of the following people. - -- [Teemperor](https://github.com/Teemperor) implemented CMake support and lcov integration, realized escape and Unicode handling in the string parser, and fixed the JSON serialization. -- [elliotgoodrich](https://github.com/elliotgoodrich) fixed an issue with double deletion in the iterator classes. -- [kirkshoop](https://github.com/kirkshoop) made the iterators of the class composable to other libraries. -- [wancw](https://github.com/wanwc) fixed a bug that hindered the class to compile with Clang. -- Tomas Åblad found a bug in the iterator implementation. -- [Joshua C. Randall](https://github.com/jrandall) fixed a bug in the floating-point serialization. -- [Aaron Burghardt](https://github.com/aburgh) implemented code to parse streams incrementally. Furthermore, he greatly improved the parser class by allowing the definition of a filter function to discard undesired elements while parsing. -- [Daniel Kopeček](https://github.com/dkopecek) fixed a bug in the compilation with GCC 5.0. -- [Florian Weber](https://github.com/Florianjw) fixed a bug in and improved the performance of the comparison operators. -- [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. -- [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. -- [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. -- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. -- [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. -- [dariomt](https://github.com/dariomt) fixed some typos in the examples. -- [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. -- [Colin Hirsch](https://github.com/ColinH) took care of a small namespace issue. -- [Huu Nguyen](https://github.com/whoshuu) correct a variable name in the documentation. -- [Silverweed](https://github.com/silverweed) overloaded `parse()` to accept an rvalue reference. -- [dariomt](https://github.com/dariomt) fixed a subtlety in MSVC type support and implemented the `get_ref()` function to get a reference to stored values. -- [ZahlGraf](https://github.com/ZahlGraf) added a workaround that allows compilation using Android NDK. -- [whackashoe](https://github.com/whackashoe) replaced a function that was marked as unsafe by Visual Studio. -- [406345](https://github.com/406345) fixed two small warnings. -- [Glen Fernandes](https://github.com/glenfe) noted a potential portability problem in the `has_mapped_type` function. -- [Corbin Hughes](https://github.com/nibroc) fixed some typos in the contribution guidelines. -- [twelsby](https://github.com/twelsby) fixed the array subscript operator, an issue that failed the MSVC build, and floating-point parsing/dumping. He further added support for unsigned integer numbers and implemented better roundtrip support for parsed numbers. -- [Volker Diels-Grabsch](https://github.com/vog) fixed a link in the README file. -- [msm-](https://github.com/msm-) added support for american fuzzy lop. -- [Annihil](https://github.com/Annihil) fixed an example in the README file. -- [Themercee](https://github.com/Themercee) noted a wrong URL in the README file. -- [Lv Zheng](https://github.com/lv-zheng) fixed a namespace issue with `int64_t` and `uint64_t`. -- [abc100m](https://github.com/abc100m) analyzed the issues with GCC 4.8 and proposed a [partial solution](https://github.com/nlohmann/json/pull/212). -- [zewt](https://github.com/zewt) added useful notes to the README file about Android. -- [Róbert Márki](https://github.com/robertmrk) added a fix to use move iterators and improved the integration via CMake. -- [Chris Kitching](https://github.com/ChrisKitching) cleaned up the CMake files. -- [Tom Needham](https://github.com/06needhamt) fixed a subtle bug with MSVC 2015 which was also proposed by [Michael K.](https://github.com/Epidal). -- [Mário Feroldi](https://github.com/thelostt) fixed a small typo. -- [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. -- [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - -Thanks a lot for helping out! - - -## Notes - -- The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). -- As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions. - - -## Execute unit tests - -To compile and run the tests, you need to execute - -```sh -$ make -$ ./json_unit "*" - -=============================================================================== -All tests passed (8905012 assertions in 32 test cases) -``` - -For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). diff --git a/ext/offbase/json/json.hpp b/ext/offbase/json/json.hpp deleted file mode 100644 index 878fb899..00000000 --- a/ext/offbase/json/json.hpp +++ /dev/null @@ -1,10435 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.0.2 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -Copyright (c) 2013-2016 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// exclude unsupported compilers -#if defined(__clang__) - #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) - #if CLANG_VERSION < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#elif defined(__GNUC__) - #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - #if GCC_VERSION < 40900 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ - - -/*! -@brief unnamed namespace with internal helper functions -@since version 1.0.0 -*/ -namespace -{ -/*! -@brief Helper to determine whether there's a key_type for T. - -Thus helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. - -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0 -*/ -template -struct has_mapped_type -{ - private: - template static char test(typename C::mapped_type*); - template static char (&test(...))[2]; - public: - static constexpr bool value = sizeof(test(0)) == 1; -}; - -/*! -@brief helper class to create locales with decimal point - -This struct is used a default locale during the JSON serialization. JSON -requires the decimal point to be `.`, so this function overloads the -`do_decimal_point()` function to return `.`. This function is called by -float-to-string conversions to retrieve the decimal separator between integer -and fractional parts. - -@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315 -@since version 2.0.0 -*/ -struct DecimalSeparator : std::numpunct -{ - char do_decimal_point() const - { - return '.'; - } -}; - -} - -/*! -@brief a class to store JSON values - -@tparam ObjectType type for JSON objects (`std::map` by default; will be used -in @ref object_t) -@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used -in @ref array_t) -@tparam StringType type for JSON strings and object keys (`std::string` by -default; will be used in @ref string_t) -@tparam BooleanType type for JSON booleans (`bool` by default; will be used -in @ref boolean_t) -@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by -default; will be used in @ref number_integer_t) -@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c -`uint64_t` by default; will be used in @ref number_unsigned_t) -@tparam NumberFloatType type for JSON floating-point numbers (`double` by -default; will be used in @ref number_float_t) -@tparam AllocatorType type of the allocator to use (`std::allocator` by -default) - -@requirement The class satisfies the following concept requirements: -- Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): - JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): - A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): - A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): - A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): - A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): - JSON values can be destructed. -- Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): - JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): - All non-static data members are private and standard layout types, the class - has no virtual functions or (virtual) base classes. -- Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): - JSON values can be compared with `==`, see @ref - operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): - JSON values can be compared with `<`, see @ref - operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): - Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of - other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): - JSON values can be compared against `std::nullptr_t` objects which are used - to model the `null` value. -- Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): - JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); - JSON values can be used like STL containers and provide reverse iterator - access. - -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. -The invariants are checked by member function assert_invariant(). - -@internal -@note ObjectType trick from http://stackoverflow.com/a/9860911 -@endinternal - -@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange -Format](http://rfc7159.net/rfc7159) - -@since version 1.0.0 - -@nosubgrouping -*/ -template < - template class ObjectType = std::map, - template class ArrayType = std::vector, - class StringType = std::string, - class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator - > -class basic_json -{ - private: - /// workaround type for MSVC - using basic_json_t = basic_json; - - public: - // forward declarations - template class json_reverse_iterator; - class json_pointer; - - ///////////////////// - // container types // - ///////////////////// - - /// @name container types - /// The canonic container types to use @ref basic_json like any other STL - /// container. - /// @{ - - /// the type of elements in a basic_json container - using value_type = basic_json; - - /// the type of an element reference - using reference = value_type&; - /// the type of an element const reference - using const_reference = const value_type&; - - /// a type to represent differences between iterators - using difference_type = std::ptrdiff_t; - /// a type to represent container sizes - using size_type = std::size_t; - - /// the allocator type - using allocator_type = AllocatorType; - - /// the type of an element pointer - using pointer = typename std::allocator_traits::pointer; - /// the type of an element const pointer - using const_pointer = typename std::allocator_traits::const_pointer; - - /// an iterator for a basic_json container - class iterator; - /// a const iterator for a basic_json container - class const_iterator; - /// a reverse iterator for a basic_json container - using reverse_iterator = json_reverse_iterator; - /// a const reverse iterator for a basic_json container - using const_reverse_iterator = json_reverse_iterator; - - /// @} - - - /*! - @brief returns the allocator associated with the container - */ - static allocator_type get_allocator() - { - return allocator_type(); - } - - - /////////////////////////// - // JSON value data types // - /////////////////////////// - - /// @name JSON value data types - /// The data types to store a JSON value. These types are derived from - /// the template arguments passed to class @ref basic_json. - /// @{ - - /*! - @brief a type for an object - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: - > An object is an unordered collection of zero or more name/value pairs, - > where a name is a string and a value is a string, number, boolean, null, - > object, or array. - - To store objects in C++, a type is defined by the template parameters - described below. - - @tparam ObjectType the container to store objects (e.g., `std::map` or - `std::unordered_map`) - @tparam StringType the type of the keys or names (e.g., `std::string`). - The comparison function `std::less` is used to order elements - inside the container. - @tparam AllocatorType the allocator to use for objects (e.g., - `std::allocator`) - - #### Default type - - With the default values for @a ObjectType (`std::map`), @a StringType - (`std::string`), and @a AllocatorType (`std::allocator`), the default - value for @a object_t is: - - @code {.cpp} - std::map< - std::string, // key_type - basic_json, // value_type - std::less, // key_compare - std::allocator> // allocator_type - > - @endcode - - #### Behavior - - The choice of @a object_t influences the behavior of the JSON class. With - the default type, objects have the following behavior: - - - When all names are unique, objects will be interoperable in the sense - that all software implementations receiving that object will agree on - the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. - - Internally, name/value pairs are stored in lexicographical order of the - names. Objects will also be serialized (see @ref dump) in this order. - For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored - and serialized as `{"a": 2, "b": 1}`. - - When comparing objects, the order of the name/value pairs is irrelevant. - This makes objects interoperable in the sense that they will not be - affected by these differences. For instance, `{"b": 1, "a": 2}` and - `{"a": 2, "b": 1}` will be treated as equal. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the object's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON object. - - #### Storage - - Objects are stored as pointers in a @ref basic_json type. That is, for any - access to object values, a pointer of type `object_t*` must be - dereferenced. - - @sa @ref array_t -- type for an array value - - @since version 1.0.0 - - @note The order name/value pairs are added to the object is *not* - preserved by the library. Therefore, iterating an object may return - name/value pairs in a different order than they were originally stored. In - fact, keys will be traversed in alphabetical order as `std::map` with - `std::less` is used by default. Please note this behavior conforms to [RFC - 7159](http://rfc7159.net/rfc7159), because any order implements the - specified "unordered" nature of JSON objects. - */ - using object_t = ObjectType, - AllocatorType>>; - - /*! - @brief a type for an array - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: - > An array is an ordered sequence of zero or more values. - - To store objects in C++, a type is defined by the template parameters - explained below. - - @tparam ArrayType container type to store arrays (e.g., `std::vector` or - `std::list`) - @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) - - #### Default type - - With the default values for @a ArrayType (`std::vector`) and @a - AllocatorType (`std::allocator`), the default value for @a array_t is: - - @code {.cpp} - std::vector< - basic_json, // value_type - std::allocator // allocator_type - > - @endcode - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the array's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON array. - - #### Storage - - Arrays are stored as pointers in a @ref basic_json type. That is, for any - access to array values, a pointer of type `array_t*` must be dereferenced. - - @sa @ref object_t -- type for an object value - - @since version 1.0.0 - */ - using array_t = ArrayType>; - - /*! - @brief a type for a string - - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: - > A string is a sequence of zero or more Unicode characters. - - To store objects in C++, a type is defined by the template parameter - described below. Unicode values are split by the JSON class into - byte-sized characters during deserialization. - - @tparam StringType the container to store strings (e.g., `std::string`). - Note this container is used for keys/names in objects, see @ref object_t. - - #### Default type - - With the default values for @a StringType (`std::string`), the default - value for @a string_t is: - - @code {.cpp} - std::string - @endcode - - #### String comparison - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > Software implementations are typically required to test names of object - > members for equality. Implementations that transform the textual - > representation into sequences of Unicode code units and then perform the - > comparison numerically, code unit by code unit, are interoperable in the - > sense that implementations will agree in all cases on equality or - > inequality of two strings. For example, implementations that compare - > strings with escaped characters unconverted may incorrectly find that - > `"a\\b"` and `"a\u005Cb"` are not equal. - - This implementation is interoperable as it does compare strings code unit - by code unit. - - #### Storage - - String values are stored as pointers in a @ref basic_json type. That is, - for any access to string values, a pointer of type `string_t*` must be - dereferenced. - - @since version 1.0.0 - */ - using string_t = StringType; - - /*! - @brief a type for a boolean - - [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a - type which differentiates the two literals `true` and `false`. - - To store objects in C++, a type is defined by the template parameter @a - BooleanType which chooses the type to use. - - #### Default type - - With the default values for @a BooleanType (`bool`), the default value for - @a boolean_t is: - - @code {.cpp} - bool - @endcode - - #### Storage - - Boolean values are stored directly inside a @ref basic_json type. - - @since version 1.0.0 - */ - using boolean_t = BooleanType; - - /*! - @brief a type for a number (integer) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store integer numbers in C++, a type is defined by the template - parameter @a NumberIntegerType which chooses the type to use. - - #### Default type - - With the default values for @a NumberIntegerType (`int64_t`), the default - value for @a number_integer_t is: - - @code {.cpp} - int64_t - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. - - When the default type is used, the maximal integer number that can be - stored is `9223372036854775807` (INT64_MAX) and the minimal integer number - that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers - that are out of range will yield over/underflow when used in a - constructor. During deserialization, too large or small integer numbers - will be automatically be stored as @ref number_unsigned_t or @ref - number_float_t. - - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. - - As this range is a subrange of the exactly supported range [INT64_MIN, - INT64_MAX], this class's integer type is interoperable. - - #### Storage - - Integer number values are stored directly inside a @ref basic_json type. - - @sa @ref number_float_t -- type for number values (floating-point) - - @sa @ref number_unsigned_t -- type for number values (unsigned integer) - - @since version 1.0.0 - */ - using number_integer_t = NumberIntegerType; - - /*! - @brief a type for a number (unsigned) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store unsigned integer numbers in C++, a type is defined by the - template parameter @a NumberUnsignedType which chooses the type to use. - - #### Default type - - With the default values for @a NumberUnsignedType (`uint64_t`), the - default value for @a number_unsigned_t is: - - @code {.cpp} - uint64_t - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. - - When the default type is used, the maximal integer number that can be - stored is `18446744073709551615` (UINT64_MAX) and the minimal integer - number that can be stored is `0`. Integer numbers that are out of range - will yield over/underflow when used in a constructor. During - deserialization, too large or small integer numbers will be automatically - be stored as @ref number_integer_t or @ref number_float_t. - - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. - - As this range is a subrange (when considered in conjunction with the - number_integer_t type) of the exactly supported range [0, UINT64_MAX], - this class's integer type is interoperable. - - #### Storage - - Integer number values are stored directly inside a @ref basic_json type. - - @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref number_integer_t -- type for number values (integer) - - @since version 2.0.0 - */ - using number_unsigned_t = NumberUnsignedType; - - /*! - @brief a type for a number (floating-point) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store floating-point numbers in C++, a type is defined by the template - parameter @a NumberFloatType which chooses the type to use. - - #### Default type - - With the default values for @a NumberFloatType (`double`), the default - value for @a number_float_t is: - - @code {.cpp} - double - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in floating-point literals will be ignored. Internally, - the value will be stored as decimal number. For instance, the C++ - floating-point literal `01.2` will be serialized to `1.2`. During - deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > This specification allows implementations to set limits on the range and - > precision of numbers accepted. Since software that implements IEEE - > 754-2008 binary64 (double precision) numbers is generally available and - > widely used, good interoperability can be achieved by implementations - > that expect no more precision or range than these provide, in the sense - > that implementations will approximate JSON numbers within the expected - > precision. - - This implementation does exactly follow this approach, as it uses double - precision floating-point numbers. Note values smaller than - `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` - will be stored as NaN internally and be serialized to `null`. - - #### Storage - - Floating-point number values are stored directly inside a @ref basic_json - type. - - @sa @ref number_integer_t -- type for number values (integer) - - @sa @ref number_unsigned_t -- type for number values (unsigned integer) - - @since version 1.0.0 - */ - using number_float_t = NumberFloatType; - - /// @} - - - /////////////////////////// - // JSON type enumeration // - /////////////////////////// - - /*! - @brief the JSON type enumeration - - This enumeration collects the different JSON types. It is internally used - to distinguish the stored values, and the functions @ref is_null(), @ref - is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref - is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and - @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and - @ref is_structured() rely on it. - - @note There are three enumeration entries (number_integer, - number_unsigned, and number_float), because the library distinguishes - these three types for numbers: @ref number_unsigned_t is used for unsigned - integers, @ref number_integer_t is used for signed integers, and @ref - number_float_t is used for floating-point numbers or to approximate - integers which do not fit in the limits of their respective type. - - @sa @ref basic_json(const value_t value_type) -- create a JSON value with - the default value for a given type - - @since version 1.0.0 - */ - enum class value_t : uint8_t - { - null, ///< null value - object, ///< object (unordered set of name/value pairs) - array, ///< array (ordered collection of values) - string, ///< string value - boolean, ///< boolean value - number_integer, ///< number value (signed integer) - number_unsigned, ///< number value (unsigned integer) - number_float, ///< number value (floating-point) - discarded ///< discarded by the the parser callback function - }; - - - private: - - /// helper for exception-safe object creation - template - static T* create(Args&& ... args) - { - AllocatorType alloc; - auto deleter = [&](T * object) - { - alloc.deallocate(object, 1); - }; - std::unique_ptr object(alloc.allocate(1), deleter); - alloc.construct(object.get(), std::forward(args)...); - assert(object.get() != nullptr); - return object.release(); - } - - //////////////////////// - // JSON value storage // - //////////////////////// - - /*! - @brief a JSON value - - The actual storage for a JSON value of the @ref basic_json class. This - union combines the different storage types for the JSON value types - defined in @ref value_t. - - JSON type | value_t type | used type - --------- | --------------- | ------------------------ - object | object | pointer to @ref object_t - array | array | pointer to @ref array_t - string | string | pointer to @ref string_t - boolean | boolean | @ref boolean_t - number | number_integer | @ref number_integer_t - number | number_unsigned | @ref number_unsigned_t - number | number_float | @ref number_float_t - null | null | *no value is stored* - - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. - - @since version 1.0.0 - */ - union json_value - { - /// object (stored with pointer to save storage) - object_t* object; - /// array (stored with pointer to save storage) - array_t* array; - /// string (stored with pointer to save storage) - string_t* string; - /// boolean - boolean_t boolean; - /// number (integer) - number_integer_t number_integer; - /// number (unsigned integer) - number_unsigned_t number_unsigned; - /// number (floating-point) - number_float_t number_float; - - /// default constructor (for null values) - json_value() = default; - /// constructor for booleans - json_value(boolean_t v) noexcept : boolean(v) {} - /// constructor for numbers (integer) - json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} - /// constructor for numbers (floating-point) - json_value(number_float_t v) noexcept : number_float(v) {} - /// constructor for empty values of a given type - json_value(value_t t) - { - switch (t) - { - case value_t::object: - { - object = create(); - break; - } - - case value_t::array: - { - array = create(); - break; - } - - case value_t::string: - { - string = create(""); - break; - } - - case value_t::boolean: - { - boolean = boolean_t(false); - break; - } - - case value_t::number_integer: - { - number_integer = number_integer_t(0); - break; - } - - case value_t::number_unsigned: - { - number_unsigned = number_unsigned_t(0); - break; - } - - case value_t::number_float: - { - number_float = number_float_t(0.0); - break; - } - - default: - { - break; - } - } - } - - /// constructor for strings - json_value(const string_t& value) - { - string = create(value); - } - - /// constructor for objects - json_value(const object_t& value) - { - object = create(value); - } - - /// constructor for arrays - json_value(const array_t& value) - { - array = create(value); - } - }; - - /*! - @brief checks the class invariants - - This function asserts the class invariants. It needs to be called at the - end of every constructor to make sure that created objects respect the - invariant. Furthermore, it has to be called each time the type of a JSON - value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. - */ - void assert_invariant() const - { - assert(m_type != value_t::object or m_value.object != nullptr); - assert(m_type != value_t::array or m_value.array != nullptr); - assert(m_type != value_t::string or m_value.string != nullptr); - } - - public: - ////////////////////////// - // JSON parser callback // - ////////////////////////// - - /*! - @brief JSON callback events - - This enumeration lists the parser events that can trigger calling a - callback function of type @ref parser_callback_t during parsing. - - @image html callback_events.png "Example when certain parse events are triggered" - - @since version 1.0.0 - */ - enum class parse_event_t : uint8_t - { - /// the parser read `{` and started to process a JSON object - object_start, - /// the parser read `}` and finished processing a JSON object - object_end, - /// the parser read `[` and started to process a JSON array - array_start, - /// the parser read `]` and finished processing a JSON array - array_end, - /// the parser read a key of a value in an object - key, - /// the parser finished reading a JSON value - value - }; - - /*! - @brief per-element parser callback type - - With a parser callback function, the result of parsing a JSON text can be - influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const string_t&, const parser_callback_t), - it is called on certain events (passed as @ref parse_event_t via parameter - @a event) with a set recursion depth @a depth and context JSON value - @a parsed. The return value of the callback function is a boolean - indicating whether the element that emitted the callback shall be kept or - not. - - We distinguish six scenarios (determined by the event type) in which the - callback function can be called. The following table describes the values - of the parameters @a depth, @a event, and @a parsed. - - parameter @a event | description | parameter @a depth | parameter @a parsed - ------------------ | ----------- | ------------------ | ------------------- - parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded - parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key - parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object - parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded - parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array - parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value - - @image html callback_events.png "Example when certain parse events are triggered" - - Discarding a value (i.e., returning `false`) has different effects - depending on the context in which function was called: - - - Discarded values in structured types are skipped. That is, the parser - will behave as if the discarded value was never read. - - In case a value outside a structured type is skipped, it is replaced - with `null`. This case happens if the top-level element is skipped. - - @param[in] depth the depth of the recursion during parsing - - @param[in] event an event of type parse_event_t indicating the context in - the callback function has been called - - @param[in,out] parsed the current intermediate parse result; note that - writing to this value has no effect for parse_event_t::key events - - @return Whether the JSON value which called the function during parsing - should be kept (`true`) or not (`false`). In the latter case, it is either - skipped completely or replaced by an empty discarded object. - - @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const string_t&, parser_callback_t) for examples - - @since version 1.0.0 - */ - using parser_callback_t = std::function; - - - ////////////////// - // constructors // - ////////////////// - - /// @name constructors and destructors - /// Constructors of class @ref basic_json, copy/move constructor, copy - /// assignment, static functions creating objects, and the destructor. - /// @{ - - /*! - @brief create an empty value with a given type - - Create an empty JSON value with a given type. The value will be default - initialized with an empty value which depends on the type: - - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` - - @param[in] value_type the type of the value to create - - @complexity Constant. - - @throw std::bad_alloc if allocation for object, array, or string value - fails - - @liveexample{The following code shows the constructor for different @ref - value_t values,basic_json__value_t} - - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - @sa @ref basic_json(boolean_t value) -- create a boolean value - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const object_t&) -- create a object value - @sa @ref basic_json(const array_t&) -- create a array value - @sa @ref basic_json(const number_float_t) -- create a number - (floating-point) value - @sa @ref basic_json(const number_integer_t) -- create a number (integer) - value - @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) - value - - @since version 1.0.0 - */ - basic_json(const value_t value_type) - : m_type(value_type), m_value(value_type) - { - assert_invariant(); - } - - /*! - @brief create a null object (implicitly) - - Create a `null` JSON value. This is the implicit version of the `null` - value constructor as it takes no parameters. - - @note The class invariant is satisfied, because it poses no requirements - for null values. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - As postcondition, it holds: `basic_json().empty() == true`. - - @liveexample{The following code shows the constructor for a `null` JSON - value.,basic_json} - - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - - @since version 1.0.0 - */ - basic_json() = default; - - /*! - @brief create a null object (explicitly) - - Create a `null` JSON value. This is the explicitly version of the `null` - value constructor as it takes a null pointer as parameter. It allows to - create `null` values by explicitly assigning a `nullptr` to a JSON value. - The passed null pointer itself is not read -- it is only used to choose - the right constructor. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. - - @liveexample{The following code shows the constructor with null pointer - parameter.,basic_json__nullptr_t} - - @sa @ref basic_json() -- default constructor (implicitly creating a `null` - value) - - @since version 1.0.0 - */ - basic_json(std::nullptr_t) noexcept - : basic_json(value_t::null) - { - assert_invariant(); - } - - /*! - @brief create an object (explicit) - - Create an object JSON value with a given content. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails - - @liveexample{The following code shows the constructor with an @ref - object_t parameter.,basic_json__object_t} - - @sa @ref basic_json(const CompatibleObjectType&) -- create an object value - from a compatible STL container - - @since version 1.0.0 - */ - basic_json(const object_t& val) - : m_type(value_t::object), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an object (implicit) - - Create an object JSON value with a given content. This constructor allows - any type @a CompatibleObjectType that can be used to construct values of - type @ref object_t. - - @tparam CompatibleObjectType An object type whose `key_type` and - `value_type` is compatible to @ref object_t. Examples include `std::map`, - `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with - a `key_type` of `std::string`, and a `value_type` from which a @ref - basic_json value can be constructed. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails - - @liveexample{The following code shows the constructor with several - compatible object type parameters.,basic_json__CompatibleObjectType} - - @sa @ref basic_json(const object_t&) -- create an object value - - @since version 1.0.0 - */ - template ::value and - std::is_constructible::value, int>::type - = 0> - basic_json(const CompatibleObjectType& val) - : m_type(value_t::object) - { - using std::begin; - using std::end; - m_value.object = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create an array (explicit) - - Create an array JSON value with a given content. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with an @ref array_t - parameter.,basic_json__array_t} - - @sa @ref basic_json(const CompatibleArrayType&) -- create an array value - from a compatible STL containers - - @since version 1.0.0 - */ - basic_json(const array_t& val) - : m_type(value_t::array), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an array (implicit) - - Create an array JSON value with a given content. This constructor allows - any type @a CompatibleArrayType that can be used to construct values of - type @ref array_t. - - @tparam CompatibleArrayType An object type whose `value_type` is - compatible to @ref array_t. Examples include `std::vector`, `std::deque`, - `std::list`, `std::forward_list`, `std::array`, `std::set`, - `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a - `value_type` from which a @ref basic_json value can be constructed. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with several - compatible array type parameters.,basic_json__CompatibleArrayType} - - @sa @ref basic_json(const array_t&) -- create an array value - - @since version 1.0.0 - */ - template ::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type - = 0> - basic_json(const CompatibleArrayType& val) - : m_type(value_t::array) - { - using std::begin; - using std::end; - m_value.array = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create an string JSON value with a given content. - - @param[in] val a value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with an @ref - string_t parameter.,basic_json__string_t} - - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const string_t& val) - : m_type(value_t::string), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create a string JSON value with a given content. - - @param[in] val a literal value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with string literal - parameter.,basic_json__string_t_value_type} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const typename string_t::value_type* val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a string (implicit) - - Create a string JSON value with a given content. - - @param[in] val a value for the string - - @tparam CompatibleStringType an string type which is compatible to @ref - string_t, for instance `std::string`. - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the construction of a string value - from a compatible type.,basic_json__CompatibleStringType} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - - @since version 1.0.0 - */ - template ::value, int>::type - = 0> - basic_json(const CompatibleStringType& val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a boolean (explicit) - - Creates a JSON boolean type from a given value. - - @param[in] val a boolean value to store - - @complexity Constant. - - @liveexample{The example below demonstrates boolean - values.,basic_json__boolean_t} - - @since version 1.0.0 - */ - basic_json(boolean_t val) noexcept - : m_type(value_t::boolean), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number (explicit) - - Create an integer number JSON value with a given content. - - @tparam T A helper type to remove this function via SFINAE in case @ref - number_integer_t is the same as `int`. In this case, this constructor - would have the same signature as @ref basic_json(const int value). Note - the helper type @a T is not visible in this constructor's interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value.,basic_json__number_integer_t} - - @sa @ref basic_json(const int) -- create a number value (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - template::value) - and std::is_same::value - , int>::type - = 0> - basic_json(const number_integer_t val) noexcept - : m_type(value_t::number_integer), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number from an enum type (explicit) - - Create an integer number JSON value with a given content. - - @param[in] val an integer to create a JSON number from - - @note This constructor allows to pass enums directly to a constructor. As - C++ has no way of specifying the type of an anonymous enum explicitly, we - can only rely on the fact that such values implicitly convert to int. As - int may already be the same type of number_integer_t, we may need to - switch off the constructor @ref basic_json(const number_integer_t). - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value from an anonymous enum.,basic_json__const_int} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const int val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an integer number (implicit) - - Create an integer number JSON value with a given content. This constructor - allows any type @a CompatibleNumberIntegerType that can be used to - construct values of type @ref number_integer_t. - - @tparam CompatibleNumberIntegerType An integer type which is compatible to - @ref number_integer_t. Examples include the types `int`, `int32_t`, - `long`, and `short`. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of several integer - number values from compatible - types.,basic_json__CompatibleIntegerNumberType} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const int) -- create a number value (integer) - - @since version 1.0.0 - */ - template::value and - std::numeric_limits::is_integer and - std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type - = 0> - basic_json(const CompatibleNumberIntegerType val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an unsigned integer number (explicit) - - Create an unsigned integer number JSON value with a given content. - - @tparam T helper type to compare number_unsigned_t and unsigned int (not - visible in) the interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number - value (unsigned integer) from a compatible number type - - @since version 2.0.0 - */ - template::value) - and std::is_same::value - , int>::type - = 0> - basic_json(const number_unsigned_t val) noexcept - : m_type(value_t::number_unsigned), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an unsigned number (implicit) - - Create an unsigned number JSON value with a given content. This - constructor allows any type @a CompatibleNumberUnsignedType that can be - used to construct values of type @ref number_unsigned_t. - - @tparam CompatibleNumberUnsignedType An integer type which is compatible - to @ref number_unsigned_t. Examples may include the types `unsigned int`, - `uint32_t`, or `unsigned short`. - - @param[in] val an unsigned integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const number_unsigned_t) -- create a number value - (unsigned) - - @since version 2.0.0 - */ - template ::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type - = 0> - basic_json(const CompatibleNumberUnsignedType val) noexcept - : m_type(value_t::number_unsigned), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create a floating-point number (explicit) - - Create a floating-point number JSON value with a given content. - - @param[in] val a floating-point value to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is created - instead. - - @complexity Constant. - - @liveexample{The following example creates several floating-point - values.,basic_json__number_float_t} - - @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number - value (floating-point) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const number_float_t val) noexcept - : m_type(value_t::number_float), m_value(val) - { - // replace infinity and NAN by null - if (not std::isfinite(val)) - { - m_type = value_t::null; - m_value = json_value(); - } - - assert_invariant(); - } - - /*! - @brief create an floating-point number (implicit) - - Create an floating-point number JSON value with a given content. This - constructor allows any type @a CompatibleNumberFloatType that can be used - to construct values of type @ref number_float_t. - - @tparam CompatibleNumberFloatType A floating-point type which is - compatible to @ref number_float_t. Examples may include the types `float` - or `double`. - - @param[in] val a floating-point to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is - created instead. - - @complexity Constant. - - @liveexample{The example below shows the construction of several - floating-point number values from compatible - types.,basic_json__CompatibleNumberFloatType} - - @sa @ref basic_json(const number_float_t) -- create a number value - (floating-point) - - @since version 1.0.0 - */ - template::value and - std::is_floating_point::value>::type - > - basic_json(const CompatibleNumberFloatType val) noexcept - : basic_json(number_float_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a container (array or object) from an initializer list - - Creates a JSON value of type array or object from the passed initializer - list @a init. In case @a type_deduction is `true` (default), the type of - the JSON value to be created is deducted from the initializer list @a init - according to the following rules: - - 1. If the list is empty, an empty JSON object value `{}` is created. - 2. If the list consists of pairs whose first element is a string, a JSON - object value is created where the first elements of the pairs are - treated as keys and the second elements are as values. - 3. In all other cases, an array is created. - - The rules aim to create the best fit between a C++ initializer list and - JSON values. The rationale is as follows: - - 1. The empty initializer list is written as `{}` which is exactly an empty - JSON object. - 2. C++ has now way of describing mapped types other than to list a list of - pairs. As JSON requires that keys must be of type string, rule 2 is the - weakest constraint one can pose on initializer lists to interpret them - as an object. - 3. In all other cases, the initializer list could not be interpreted as - JSON object type, so interpreting it as JSON array type is safe. - - With the rules described above, the following JSON values cannot be - expressed by an initializer list: - - - the empty array (`[]`): use @ref array(std::initializer_list) - with an empty initializer list in this case - - arrays whose elements satisfy rule 2: use @ref - array(std::initializer_list) with the same initializer list - in this case - - @note When used without parentheses around an empty initializer list, @ref - basic_json() is called instead of this function, yielding the JSON null - value. - - @param[in] init initializer list with JSON values - - @param[in] type_deduction internal parameter; when set to `true`, the type - of the JSON value is deducted from the initializer list @a init; when set - to `false`, the type provided via @a manual_type is forced. This mode is - used by the functions @ref array(std::initializer_list) and - @ref object(std::initializer_list). - - @param[in] manual_type internal parameter; when @a type_deduction is set - to `false`, the created JSON value will use the provided type (only @ref - value_t::array and @ref value_t::object are valid); when @a type_deduction - is set to `true`, this parameter has no effect - - @throw std::domain_error if @a type_deduction is `false`, @a manual_type - is `value_t::object`, but @a init contains an element which is not a pair - whose first element is a string; example: `"cannot create object from - initializer list"` - - @complexity Linear in the size of the initializer list @a init. - - @liveexample{The example below shows how JSON values are created from - initializer lists.,basic_json__list_init_t} - - @sa @ref array(std::initializer_list) -- create a JSON array - value from an initializer list - @sa @ref object(std::initializer_list) -- create a JSON object - value from an initializer list - - @since version 1.0.0 - */ - basic_json(std::initializer_list init, - bool type_deduction = true, - value_t manual_type = value_t::array) - { - // check if each element is an array with two elements whose first - // element is a string - bool is_an_object = std::all_of(init.begin(), init.end(), - [](const basic_json & element) - { - return element.is_array() and element.size() == 2 and element[0].is_string(); - }); - - // adjust type if type deduction is not wanted - if (not type_deduction) - { - // if array is wanted, do not create an object though possible - if (manual_type == value_t::array) - { - is_an_object = false; - } - - // if object is wanted but impossible, throw an exception - if (manual_type == value_t::object and not is_an_object) - { - throw std::domain_error("cannot create object from initializer list"); - } - } - - if (is_an_object) - { - // the initializer list is a list of pairs -> create object - m_type = value_t::object; - m_value = value_t::object; - - std::for_each(init.begin(), init.end(), [this](const basic_json & element) - { - m_value.object->emplace(*(element[0].m_value.string), element[1]); - }); - } - else - { - // the initializer list describes an array -> create array - m_type = value_t::array; - m_value.array = create(init); - } - - assert_invariant(); - } - - /*! - @brief explicitly create an array from an initializer list - - Creates a JSON array value from a given initializer list. That is, given a - list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the - initializer list is empty, the empty array `[]` is created. - - @note This function is only needed to express two edge cases that cannot - be realized with the initializer list constructor (@ref - basic_json(std::initializer_list, bool, value_t)). These cases - are: - 1. creating an array whose elements are all pairs whose first element is a - string -- in this case, the initializer list constructor would create an - object, taking the first elements as keys - 2. creating an empty array -- passing the empty initializer list to the - initializer list constructor yields an empty object - - @param[in] init initializer list with JSON values to create an array from - (optional) - - @return JSON array value - - @complexity Linear in the size of @a init. - - @liveexample{The following code shows an example for the `array` - function.,array} - - @sa @ref basic_json(std::initializer_list, bool, value_t) -- - create a JSON value from an initializer list - @sa @ref object(std::initializer_list) -- create a JSON object - value from an initializer list - - @since version 1.0.0 - */ - static basic_json array(std::initializer_list init = - std::initializer_list()) - { - return basic_json(init, false, value_t::array); - } - - /*! - @brief explicitly create an object from an initializer list - - Creates a JSON object value from a given initializer list. The initializer - lists elements must be pairs, and their first elements must be strings. If - the initializer list is empty, the empty object `{}` is created. - - @note This function is only added for symmetry reasons. In contrast to the - related function @ref array(std::initializer_list), there are - no cases which can only be expressed by this function. That is, any - initializer list @a init can also be passed to the initializer list - constructor @ref basic_json(std::initializer_list, bool, - value_t). - - @param[in] init initializer list to create an object from (optional) - - @return JSON object value - - @throw std::domain_error if @a init is not a pair whose first elements are - strings; thrown by - @ref basic_json(std::initializer_list, bool, value_t) - - @complexity Linear in the size of @a init. - - @liveexample{The following code shows an example for the `object` - function.,object} - - @sa @ref basic_json(std::initializer_list, bool, value_t) -- - create a JSON value from an initializer list - @sa @ref array(std::initializer_list) -- create a JSON array - value from an initializer list - - @since version 1.0.0 - */ - static basic_json object(std::initializer_list init = - std::initializer_list()) - { - return basic_json(init, false, value_t::object); - } - - /*! - @brief construct an array with count copies of given value - - Constructs a JSON array value by creating @a cnt copies of a passed value. - In case @a cnt is `0`, an empty array is created. As postcondition, - `std::distance(begin(),end()) == cnt` holds. - - @param[in] cnt the number of JSON copies of @a val to create - @param[in] val the JSON value to copy - - @complexity Linear in @a cnt. - - @liveexample{The following code shows examples for the @ref - basic_json(size_type\, const basic_json&) - constructor.,basic_json__size_type_basic_json} - - @since version 1.0.0 - */ - basic_json(size_type cnt, const basic_json& val) - : m_type(value_t::array) - { - m_value.array = create(cnt, val); - assert_invariant(); - } - - /*! - @brief construct a JSON container given an iterator range - - Constructs the JSON value with the contents of the range `[first, last)`. - The semantics depends on the different types a JSON value can have: - - In case of primitive types (number, boolean, or string), @a first must - be `begin()` and @a last must be `end()`. In this case, the value is - copied. Otherwise, std::out_of_range is thrown. - - In case of structured types (array, object), the constructor behaves as - similar versions for `std::vector`. - - In case of a null type, std::domain_error is thrown. - - @tparam InputIT an input iterator type (@ref iterator or @ref - const_iterator) - - @param[in] first begin of the range to copy from (included) - @param[in] last end of the range to copy from (excluded) - - @pre Iterators @a first and @a last must be initialized. - - @throw std::domain_error if iterators are not compatible; that is, do not - belong to the same JSON value; example: `"iterators are not compatible"` - @throw std::out_of_range if iterators are for a primitive type (number, - boolean, or string) where an out of range error can be detected easily; - example: `"iterators out of range"` - @throw std::bad_alloc if allocation for object, array, or string fails - @throw std::domain_error if called with a null value; example: `"cannot - use construct with iterators from null"` - - @complexity Linear in distance between @a first and @a last. - - @liveexample{The example below shows several ways to create JSON values by - specifying a subrange with iterators.,basic_json__InputIt_InputIt} - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - basic_json(InputIT first, InputIT last) - { - assert(first.m_object != nullptr); - assert(last.m_object != nullptr); - - // make sure iterator fits the current value - if (first.m_object != last.m_object) - { - throw std::domain_error("iterators are not compatible"); - } - - // copy type from first iterator - m_type = first.m_object->m_type; - - // check if iterator range is complete for primitive values - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) - { - throw std::out_of_range("iterators out of range"); - } - break; - } - - default: - { - break; - } - } - - switch (m_type) - { - case value_t::number_integer: - { - m_value.number_integer = first.m_object->m_value.number_integer; - break; - } - - case value_t::number_unsigned: - { - m_value.number_unsigned = first.m_object->m_value.number_unsigned; - break; - } - - case value_t::number_float: - { - m_value.number_float = first.m_object->m_value.number_float; - break; - } - - case value_t::boolean: - { - m_value.boolean = first.m_object->m_value.boolean; - break; - } - - case value_t::string: - { - m_value = *first.m_object->m_value.string; - break; - } - - case value_t::object: - { - m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); - break; - } - - case value_t::array: - { - m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); - } - } - - assert_invariant(); - } - - /*! - @brief construct a JSON value given an input stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates constructing a JSON value from - a `std::stringstream` with and without callback - function.,basic_json__istream} - - @since version 2.0.0 - */ - explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) - { - *this = parser(i, cb).parse(); - assert_invariant(); - } - - /////////////////////////////////////// - // other constructors and destructor // - /////////////////////////////////////// - - /*! - @brief copy constructor - - Creates a copy of a given JSON value. - - @param[in] other the JSON value to copy - - @complexity Linear in the size of @a other. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - As postcondition, it holds: `other == basic_json(other)`. - - @throw std::bad_alloc if allocation for object, array, or string fails. - - @liveexample{The following code shows an example for the copy - constructor.,basic_json__basic_json} - - @since version 1.0.0 - */ - basic_json(const basic_json& other) - : m_type(other.m_type) - { - // check of passed value is valid - other.assert_invariant(); - - switch (m_type) - { - case value_t::object: - { - m_value = *other.m_value.object; - break; - } - - case value_t::array: - { - m_value = *other.m_value.array; - break; - } - - case value_t::string: - { - m_value = *other.m_value.string; - break; - } - - case value_t::boolean: - { - m_value = other.m_value.boolean; - break; - } - - case value_t::number_integer: - { - m_value = other.m_value.number_integer; - break; - } - - case value_t::number_unsigned: - { - m_value = other.m_value.number_unsigned; - break; - } - - case value_t::number_float: - { - m_value = other.m_value.number_float; - break; - } - - default: - { - break; - } - } - - assert_invariant(); - } - - /*! - @brief move constructor - - Move constructor. Constructs a JSON value with the contents of the given - value @a other using move semantics. It "steals" the resources from @a - other and leaves it as JSON null value. - - @param[in,out] other value to move to this object - - @post @a other is a JSON null value - - @complexity Constant. - - @liveexample{The code below shows the move constructor explicitly called - via std::move.,basic_json__moveconstructor} - - @since version 1.0.0 - */ - basic_json(basic_json&& other) noexcept - : m_type(std::move(other.m_type)), - m_value(std::move(other.m_value)) - { - // check that passed value is valid - other.assert_invariant(); - - // invalidate payload - other.m_type = value_t::null; - other.m_value = {}; - - assert_invariant(); - } - - /*! - @brief copy assignment - - Copy assignment operator. Copies a JSON value via the "copy and swap" - strategy: It is expressed in terms of the copy constructor, destructor, - and the swap() member function. - - @param[in] other value to copy from - - @complexity Linear. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - @liveexample{The code below shows and example for the copy assignment. It - creates a copy of value `a` which is then swapped with `b`. Finally\, the - copy of `a` (which is the null value after the swap) is - destroyed.,basic_json__copyassignment} - - @since version 1.0.0 - */ - reference& operator=(basic_json other) noexcept ( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - // check that passed value is valid - other.assert_invariant(); - - using std::swap; - swap(m_type, other.m_type); - swap(m_value, other.m_value); - - assert_invariant(); - return *this; - } - - /*! - @brief destructor - - Destroys the JSON value and frees all allocated memory. - - @complexity Linear. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is linear. - - All stored elements are destroyed and all memory is freed. - - @since version 1.0.0 - */ - ~basic_json() - { - assert_invariant(); - - switch (m_type) - { - case value_t::object: - { - AllocatorType alloc; - alloc.destroy(m_value.object); - alloc.deallocate(m_value.object, 1); - break; - } - - case value_t::array: - { - AllocatorType alloc; - alloc.destroy(m_value.array); - alloc.deallocate(m_value.array, 1); - break; - } - - case value_t::string: - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - break; - } - - default: - { - // all other types need no specific destructor - break; - } - } - } - - /// @} - - public: - /////////////////////// - // object inspection // - /////////////////////// - - /// @name object inspection - /// Functions to inspect the type of a JSON value. - /// @{ - - /*! - @brief serialization - - Serialization function for JSON values. The function tries to mimic - Python's `json.dumps()` function, and currently supports its @a indent - parameter. - - @param[in] indent If indent is nonnegative, then array elements and object - members will be pretty-printed with that indent level. An indent level of - `0` will only insert newlines. `-1` (the default) selects the most compact - representation. - - @return string containing the serialization of the JSON value - - @complexity Linear. - - @liveexample{The following example shows the effect of different @a indent - parameters to the result of the serialization.,dump} - - @see https://docs.python.org/2/library/json.html#json.dump - - @since version 1.0.0 - */ - string_t dump(const int indent = -1) const - { - std::stringstream ss; - // fix locale problems - ss.imbue(std::locale(std::locale(), new DecimalSeparator)); - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - ss.precision(std::numeric_limits::digits10); - - if (indent >= 0) - { - dump(ss, true, static_cast(indent)); - } - else - { - dump(ss, false, 0); - } - - return ss.str(); - } - - /*! - @brief return the type of the JSON value (explicit) - - Return the type of the JSON value as a value from the @ref value_t - enumeration. - - @return the type of the JSON value - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `type()` for all JSON - types.,type} - - @since version 1.0.0 - */ - constexpr value_t type() const noexcept - { - return m_type; - } - - /*! - @brief return whether type is primitive - - This function returns true iff the JSON type is primitive (string, number, - boolean, or null). - - @return `true` if type is primitive (string, number, boolean, or null), - `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_primitive()` for all JSON - types.,is_primitive} - - @sa @ref is_structured() -- returns whether JSON value is structured - @sa @ref is_null() -- returns whether JSON value is `null` - @sa @ref is_string() -- returns whether JSON value is a string - @sa @ref is_boolean() -- returns whether JSON value is a boolean - @sa @ref is_number() -- returns whether JSON value is a number - - @since version 1.0.0 - */ - constexpr bool is_primitive() const noexcept - { - return is_null() or is_string() or is_boolean() or is_number(); - } - - /*! - @brief return whether type is structured - - This function returns true iff the JSON type is structured (array or - object). - - @return `true` if type is structured (array or object), `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_structured()` for all JSON - types.,is_structured} - - @sa @ref is_primitive() -- returns whether value is primitive - @sa @ref is_array() -- returns whether value is an array - @sa @ref is_object() -- returns whether value is an object - - @since version 1.0.0 - */ - constexpr bool is_structured() const noexcept - { - return is_array() or is_object(); - } - - /*! - @brief return whether value is null - - This function returns true iff the JSON value is null. - - @return `true` if type is null, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_null()` for all JSON - types.,is_null} - - @since version 1.0.0 - */ - constexpr bool is_null() const noexcept - { - return m_type == value_t::null; - } - - /*! - @brief return whether value is a boolean - - This function returns true iff the JSON value is a boolean. - - @return `true` if type is boolean, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_boolean()` for all JSON - types.,is_boolean} - - @since version 1.0.0 - */ - constexpr bool is_boolean() const noexcept - { - return m_type == value_t::boolean; - } - - /*! - @brief return whether value is a number - - This function returns true iff the JSON value is a number. This includes - both integer and floating-point values. - - @return `true` if type is number (regardless whether integer, unsigned - integer or floating-type), `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number()` for all JSON - types.,is_number} - - @sa @ref is_number_integer() -- check if value is an integer or unsigned - integer number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 1.0.0 - */ - constexpr bool is_number() const noexcept - { - return is_number_integer() or is_number_float(); - } - - /*! - @brief return whether value is an integer number - - This function returns true iff the JSON value is an integer or unsigned - integer number. This excludes floating-point values. - - @return `true` if type is an integer or unsigned integer number, `false` - otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_integer()` for all - JSON types.,is_number_integer} - - @sa @ref is_number() -- check if value is a number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 1.0.0 - */ - constexpr bool is_number_integer() const noexcept - { - return m_type == value_t::number_integer or m_type == value_t::number_unsigned; - } - - /*! - @brief return whether value is an unsigned integer number - - This function returns true iff the JSON value is an unsigned integer - number. This excludes floating-point and (signed) integer values. - - @return `true` if type is an unsigned integer number, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_unsigned()` for all - JSON types.,is_number_unsigned} - - @sa @ref is_number() -- check if value is a number - @sa @ref is_number_integer() -- check if value is an integer or unsigned - integer number - @sa @ref is_number_float() -- check if value is a floating-point number - - @since version 2.0.0 - */ - constexpr bool is_number_unsigned() const noexcept - { - return m_type == value_t::number_unsigned; - } - - /*! - @brief return whether value is a floating-point number - - This function returns true iff the JSON value is a floating-point number. - This excludes integer and unsigned integer values. - - @return `true` if type is a floating-point number, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_number_float()` for all - JSON types.,is_number_float} - - @sa @ref is_number() -- check if value is number - @sa @ref is_number_integer() -- check if value is an integer number - @sa @ref is_number_unsigned() -- check if value is an unsigned integer - number - - @since version 1.0.0 - */ - constexpr bool is_number_float() const noexcept - { - return m_type == value_t::number_float; - } - - /*! - @brief return whether value is an object - - This function returns true iff the JSON value is an object. - - @return `true` if type is object, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_object()` for all JSON - types.,is_object} - - @since version 1.0.0 - */ - constexpr bool is_object() const noexcept - { - return m_type == value_t::object; - } - - /*! - @brief return whether value is an array - - This function returns true iff the JSON value is an array. - - @return `true` if type is array, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_array()` for all JSON - types.,is_array} - - @since version 1.0.0 - */ - constexpr bool is_array() const noexcept - { - return m_type == value_t::array; - } - - /*! - @brief return whether value is a string - - This function returns true iff the JSON value is a string. - - @return `true` if type is string, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_string()` for all JSON - types.,is_string} - - @since version 1.0.0 - */ - constexpr bool is_string() const noexcept - { - return m_type == value_t::string; - } - - /*! - @brief return whether value is discarded - - This function returns true iff the JSON value was discarded during parsing - with a callback function (see @ref parser_callback_t). - - @note This function will always be `false` for JSON values after parsing. - That is, discarded values can only occur during parsing, but will be - removed when inside a structured value or replaced by null in other cases. - - @return `true` if type is discarded, `false` otherwise. - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies `is_discarded()` for all JSON - types.,is_discarded} - - @since version 1.0.0 - */ - constexpr bool is_discarded() const noexcept - { - return m_type == value_t::discarded; - } - - /*! - @brief return the type of the JSON value (implicit) - - Implicitly return the type of the JSON value as a value from the @ref - value_t enumeration. - - @return the type of the JSON value - - @complexity Constant. - - @exceptionsafety No-throw guarantee: this member function never throws - exceptions. - - @liveexample{The following code exemplifies the @ref value_t operator for - all JSON types.,operator__value_t} - - @since version 1.0.0 - */ - constexpr operator value_t() const noexcept - { - return m_type; - } - - /// @} - - private: - ////////////////// - // value access // - ////////////////// - - /// get an object (explicit) - template ::value and - std::is_convertible::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_object()) - { - return T(m_value.object->begin(), m_value.object->end()); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an object (explicit) - object_t get_impl(object_t*) const - { - if (is_object()) - { - return *(m_value.object); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - T to_vector; - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not std::is_same::value - , int>::type = 0> - std::vector get_impl(std::vector*) const - { - if (is_array()) - { - std::vector to_vector; - to_vector.reserve(m_value.array->size()); - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template ::value and - not has_mapped_type::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - return T(m_value.array->begin(), m_value.array->end()); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - array_t get_impl(array_t*) const - { - if (is_array()) - { - return *(m_value.array); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get a string (explicit) - template ::value - , int>::type = 0> - T get_impl(T*) const - { - if (is_string()) - { - return *m_value.string; - } - else - { - throw std::domain_error("type must be string, but is " + type_name()); - } - } - - /// get a number (explicit) - template::value - , int>::type = 0> - T get_impl(T*) const - { - switch (m_type) - { - case value_t::number_integer: - { - return static_cast(m_value.number_integer); - } - - case value_t::number_unsigned: - { - return static_cast(m_value.number_unsigned); - } - - case value_t::number_float: - { - return static_cast(m_value.number_float); - } - - default: - { - throw std::domain_error("type must be number, but is " + type_name()); - } - } - } - - /// get a boolean (explicit) - constexpr boolean_t get_impl(boolean_t*) const - { - return is_boolean() - ? m_value.boolean - : throw std::domain_error("type must be boolean, but is " + type_name()); - } - - /// get a pointer to the value (object) - object_t* get_impl_ptr(object_t*) noexcept - { - return is_object() ? m_value.object : nullptr; - } - - /// get a pointer to the value (object) - constexpr const object_t* get_impl_ptr(const object_t*) const noexcept - { - return is_object() ? m_value.object : nullptr; - } - - /// get a pointer to the value (array) - array_t* get_impl_ptr(array_t*) noexcept - { - return is_array() ? m_value.array : nullptr; - } - - /// get a pointer to the value (array) - constexpr const array_t* get_impl_ptr(const array_t*) const noexcept - { - return is_array() ? m_value.array : nullptr; - } - - /// get a pointer to the value (string) - string_t* get_impl_ptr(string_t*) noexcept - { - return is_string() ? m_value.string : nullptr; - } - - /// get a pointer to the value (string) - constexpr const string_t* get_impl_ptr(const string_t*) const noexcept - { - return is_string() ? m_value.string : nullptr; - } - - /// get a pointer to the value (boolean) - boolean_t* get_impl_ptr(boolean_t*) noexcept - { - return is_boolean() ? &m_value.boolean : nullptr; - } - - /// get a pointer to the value (boolean) - constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept - { - return is_boolean() ? &m_value.boolean : nullptr; - } - - /// get a pointer to the value (integer number) - number_integer_t* get_impl_ptr(number_integer_t*) noexcept - { - return is_number_integer() ? &m_value.number_integer : nullptr; - } - - /// get a pointer to the value (integer number) - constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept - { - return is_number_integer() ? &m_value.number_integer : nullptr; - } - - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept - { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; - } - - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept - { - return is_number_unsigned() ? &m_value.number_unsigned : nullptr; - } - - /// get a pointer to the value (floating-point number) - number_float_t* get_impl_ptr(number_float_t*) noexcept - { - return is_number_float() ? &m_value.number_float : nullptr; - } - - /// get a pointer to the value (floating-point number) - constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept - { - return is_number_float() ? &m_value.number_float : nullptr; - } - - /*! - @brief helper function to implement get_ref() - - This funcion helps to implement get_ref() without code duplication for - const and non-const overloads - - @tparam ThisType will be deduced as `basic_json` or `const basic_json` - - @throw std::domain_error if ReferenceType does not match underlying value - type of the current JSON - */ - template - static ReferenceType get_ref_impl(ThisType& obj) - { - // helper type - using PointerType = typename std::add_pointer::type; - - // delegate the call to get_ptr<>() - auto ptr = obj.template get_ptr(); - - if (ptr != nullptr) - { - return *ptr; - } - else - { - throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + - obj.type_name()); - } - } - - public: - - /// @name value access - /// Direct access to the stored value of a JSON value. - /// @{ - - /*! - @brief get a value (explicit) - - Explicit type conversion between the JSON value and a compatible value. - - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays - - @return copy of the JSON value, converted to type @a ValueType - - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON; example: `"type must be object, but is null"` - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows several conversions from JSON values - to other types. There a few things to note: (1) Floating-point numbers can - be converted to integers\, (2) A JSON array can be converted to a standard - `std::vector`\, (3) A JSON object can be converted to C++ - associative containers such as `std::unordered_map`.,get__ValueType_const} - - @internal - The idea of using a casted null pointer to choose the correct - implementation is from . - @endinternal - - @sa @ref operator ValueType() const for implicit conversion - @sa @ref get() for pointer-member access - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - ValueType get() const - { - return get_impl(static_cast(nullptr)); - } - - /*! - @brief get a pointer value (explicit) - - Explicit pointer access to the internally stored JSON value. No copies are - made. - - @warning The pointer becomes invalid if the underlying JSON object - changes. - - @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref - object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. - - @return pointer to the internally stored JSON value if the requested - pointer type @a PointerType fits to the JSON value; `nullptr` otherwise - - @complexity Constant. - - @liveexample{The example below shows how pointers to internal values of a - JSON value can be requested. Note that no type conversions are made and a - `nullptr` is returned if the value and the requested pointer type does not - match.,get__PointerType} - - @sa @ref get_ptr() for explicit pointer-member access - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - PointerType get() noexcept - { - // delegate the call to get_ptr - return get_ptr(); - } - - /*! - @brief get a pointer value (explicit) - @copydoc get() - */ - template::value - , int>::type = 0> - constexpr const PointerType get() const noexcept - { - // delegate the call to get_ptr - return get_ptr(); - } - - /*! - @brief get a pointer value (implicit) - - Implicit pointer access to the internally stored JSON value. No copies are - made. - - @warning Writing data to the pointee of the result yields an undefined - state. - - @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref - object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. Enforced by a static - assertion. - - @return pointer to the internally stored JSON value if the requested - pointer type @a PointerType fits to the JSON value; `nullptr` otherwise - - @complexity Constant. - - @liveexample{The example below shows how pointers to internal values of a - JSON value can be requested. Note that no type conversions are made and a - `nullptr` is returned if the value and the requested pointer type does not - match.,get_ptr} - - @since version 1.0.0 - */ - template::value - , int>::type = 0> - PointerType get_ptr() noexcept - { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() - return get_impl_ptr(static_cast(nullptr)); - } - - /*! - @brief get a pointer value (implicit) - @copydoc get_ptr() - */ - template::value - and std::is_const::type>::value - , int>::type = 0> - constexpr const PointerType get_ptr() const noexcept - { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - or std::is_same::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() const - return get_impl_ptr(static_cast(nullptr)); - } - - /*! - @brief get a reference value (implicit) - - Implict reference access to the internally stored JSON value. No copies - are made. - - @warning Writing data to the referee of the result yields an undefined - state. - - @tparam ReferenceType reference type; must be a reference to @ref array_t, - @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or - @ref number_float_t. Enforced by static assertion. - - @return reference to the internally stored JSON value if the requested - reference type @a ReferenceType fits to the JSON value; throws - std::domain_error otherwise - - @throw std::domain_error in case passed type @a ReferenceType is - incompatible with the stored JSON value - - @complexity Constant. - - @liveexample{The example shows several calls to `get_ref()`.,get_ref} - - @since version 1.1.0 - */ - template::value - , int>::type = 0> - ReferenceType get_ref() - { - // delegate call to get_ref_impl - return get_ref_impl(*this); - } - - /*! - @brief get a reference value (implicit) - @copydoc get_ref() - */ - template::value - and std::is_const::type>::value - , int>::type = 0> - ReferenceType get_ref() const - { - // delegate call to get_ref_impl - return get_ref_impl(*this); - } - - /*! - @brief get a value (implicit) - - Implicit type conversion between the JSON value and a compatible value. - The call is realized by calling @ref get() const. - - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays. The character type of @ref string_t - as well as an initializer list of this type is excluded to avoid - ambiguities as these types implicitly convert to `std::string`. - - @return copy of the JSON value, converted to type @a ValueType - - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON, thrown by @ref get() const - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows several conversions from JSON values - to other types. There a few things to note: (1) Floating-point numbers can - be converted to integers\, (2) A JSON array can be converted to a standard - `std::vector`\, (3) A JSON object can be converted to C++ - associative containers such as `std::unordered_map`.,operator__ValueType} - - @since version 1.0.0 - */ - template < typename ValueType, typename - std::enable_if < - not std::is_pointer::value - and not std::is_same::value -#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 - and not std::is_same>::value -#endif - , int >::type = 0 > - operator ValueType() const - { - // delegate the call to get<>() const - return get(); - } - - /// @} - - - //////////////////// - // element access // - //////////////////// - - /// @name element access - /// Access to the JSON value. - /// @{ - - /*! - @brief access specified array element with bounds checking - - Returns a reference to the element at specified location @a idx, with - bounds checking. - - @param[in] idx index of the element to access - - @return reference to the element at index @a idx - - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read and - written using `at()`.,at__size_type} - - @since version 1.0.0 - */ - reference at(size_type idx) - { - // at only works for arrays - if (is_array()) - { - try - { - return m_value.array->at(idx); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified array element with bounds checking - - Returns a const reference to the element at specified location @a idx, - with bounds checking. - - @param[in] idx index of the element to access - - @return const reference to the element at index @a idx - - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read using - `at()`.,at__size_type_const} - - @since version 1.0.0 - */ - const_reference at(size_type idx) const - { - // at only works for arrays - if (is_array()) - { - try - { - return m_value.array->at(idx); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified object element with bounds checking - - Returns a reference to the element at with specified key @a key, with - bounds checking. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using `at()`.,at__object_t_key_type} - - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - reference at(const typename object_t::key_type& key) - { - // at only works for objects - if (is_object()) - { - try - { - return m_value.object->at(key); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified object element with bounds checking - - Returns a const reference to the element at with specified key @a key, - with bounds checking. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - `at()`.,at__object_t_key_type_const} - - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - const_reference at(const typename object_t::key_type& key) const - { - // at only works for objects - if (is_object()) - { - try - { - return m_value.object->at(key); - } - catch (std::out_of_range&) - { - // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); - } - } - else - { - throw std::domain_error("cannot use at() with " + type_name()); - } - } - - /*! - @brief access specified array element - - Returns a reference to the element at specified location @a idx. - - @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), - then the array is silently filled up with `null` values to make `idx` a - valid reference to the last stored element. - - @param[in] idx index of the element to access - - @return reference to the element at index @a idx - - @throw std::domain_error if JSON is not an array or null; example: - `"cannot use operator[] with string"` - - @complexity Constant if @a idx is in the range of the array. Otherwise - linear in `idx - size()`. - - @liveexample{The example below shows how array elements can be read and - written using `[]` operator. Note the addition of `null` - values.,operatorarray__size_type} - - @since version 1.0.0 - */ - reference operator[](size_type idx) - { - // implicitly convert null value to an empty array - if (is_null()) - { - m_type = value_t::array; - m_value.array = create(); - assert_invariant(); - } - - // operator[] only works for arrays - if (is_array()) - { - // fill up array with null values if given idx is outside range - if (idx >= m_value.array->size()) - { - m_value.array->insert(m_value.array->end(), - idx - m_value.array->size() + 1, - basic_json()); - } - - return m_value.array->operator[](idx); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified array element - - Returns a const reference to the element at specified location @a idx. - - @param[in] idx index of the element to access - - @return const reference to the element at index @a idx - - @throw std::domain_error if JSON is not an array; example: `"cannot use - operator[] with null"` - - @complexity Constant. - - @liveexample{The example below shows how array elements can be read using - the `[]` operator.,operatorarray__size_type_const} - - @since version 1.0.0 - */ - const_reference operator[](size_type idx) const - { - // const operator[] only works for arrays - if (is_array()) - { - return m_value.array->operator[](idx); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - reference operator[](const typename object_t::key_type& key) - { - // implicitly convert null value to an empty object - if (is_null()) - { - m_type = value_t::object; - m_value.object = create(); - assert_invariant(); - } - - // operator[] only works for objects - if (is_object()) - { - return m_value.object->operator[](key); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - const_reference operator[](const typename object_t::key_type& key) const - { - // const operator[] only works for objects - if (is_object()) - { - assert(m_value.object->find(key) != m_value.object->end()); - return m_value.object->find(key)->second; - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - template - reference operator[](T * (&key)[n]) - { - return operator[](static_cast(key)); - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @note This function is required for compatibility reasons with Clang. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.0.0 - */ - template - const_reference operator[](T * (&key)[n]) const - { - return operator[](static_cast(key)); - } - - /*! - @brief access specified object element - - Returns a reference to the element at with specified key @a key. - - @note If @a key is not found in the object, then it is silently added to - the object and filled with a `null` value to make `key` a valid reference. - In case the value was `null` before, it is converted to an object. - - @param[in] key key of the element to access - - @return reference to the element at key @a key - - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read and - written using the `[]` operator.,operatorarray__key_type} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.1.0 - */ - template - reference operator[](T* key) - { - // implicitly convert null to object - if (is_null()) - { - m_type = value_t::object; - m_value = value_t::object; - assert_invariant(); - } - - // at only works for objects - if (is_object()) - { - return m_value.object->operator[](key); - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief read-only access specified object element - - Returns a const reference to the element at with specified key @a key. No - bounds checking is performed. - - @warning If the element with key @a key does not exist, the behavior is - undefined. - - @param[in] key key of the element to access - - @return const reference to the element at key @a key - - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be read using - the `[]` operator.,operatorarray__key_type_const} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref value() for access by value with a default value - - @since version 1.1.0 - */ - template - const_reference operator[](T* key) const - { - // at only works for objects - if (is_object()) - { - assert(m_value.object->find(key) != m_value.object->end()); - return m_value.object->find(key)->second; - } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } - } - - /*! - @brief access specified object element with default value - - Returns either a copy of an object's element at the specified key @a key - or a given default value if no element with key @a key exists. - - The function is basically equivalent to executing - @code {.cpp} - try { - return at(key); - } catch(std::out_of_range) { - return default_value; - } - @endcode - - @note Unlike @ref at(const typename object_t::key_type&), this function - does not throw if the given key @a key was not found. - - @note Unlike @ref operator[](const typename object_t::key_type& key), this - function does not implicitly add an element to the position defined by @a - key. This function is furthermore also applicable to const objects. - - @param[in] key key of the element to access - @param[in] default_value the value to return if @a key is not found - - @tparam ValueType type compatible to JSON values, for instance `int` for - JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for - JSON arrays. Note the type of the expected value at @a key and the default - value @a default_value must be compatible. - - @return copy of the element at key @a key or @a default_value if @a key - is not found - - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be queried - with a default value.,basic_json__value} - - @sa @ref at(const typename object_t::key_type&) for access by reference - with range checking - @sa @ref operator[](const typename object_t::key_type&) for unchecked - access by reference - - @since version 1.0.0 - */ - template ::value - , int>::type = 0> - ValueType value(const typename object_t::key_type& key, ValueType default_value) const - { - // at only works for objects - if (is_object()) - { - // if key is found, return value and given default value otherwise - const auto it = find(key); - if (it != end()) - { - return *it; - } - else - { - return default_value; - } - } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } - } - - /*! - @brief overload for a default value of type const char* - @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const - */ - string_t value(const typename object_t::key_type& key, const char* default_value) const - { - return value(key, string_t(default_value)); - } - - /*! - @brief access specified object element via JSON Pointer with default value - - Returns either a copy of an object's element at the specified key @a key - or a given default value if no element with key @a key exists. - - The function is basically equivalent to executing - @code {.cpp} - try { - return at(ptr); - } catch(std::out_of_range) { - return default_value; - } - @endcode - - @note Unlike @ref at(const json_pointer&), this function does not throw - if the given key @a key was not found. - - @param[in] ptr a JSON pointer to the element to access - @param[in] default_value the value to return if @a ptr found no value - - @tparam ValueType type compatible to JSON values, for instance `int` for - JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for - JSON arrays. Note the type of the expected value at @a key and the default - value @a default_value must be compatible. - - @return copy of the element at key @a key or @a default_value if @a key - is not found - - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` - - @complexity Logarithmic in the size of the container. - - @liveexample{The example below shows how object elements can be queried - with a default value.,basic_json__value_ptr} - - @sa @ref operator[](const json_pointer&) for unchecked access by reference - - @since version 2.0.2 - */ - template ::value - , int>::type = 0> - ValueType value(const json_pointer& ptr, ValueType default_value) const - { - // at only works for objects - if (is_object()) - { - // if pointer resolves a value, return it or use default value - try - { - return ptr.get_checked(this); - } - catch (std::out_of_range&) - { - return default_value; - } - } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } - } - - /*! - @brief overload for a default value of type const char* - @copydoc basic_json::value(const json_pointer&, ValueType) const - */ - string_t value(const json_pointer& ptr, const char* default_value) const - { - return value(ptr, string_t(default_value)); - } - - /*! - @brief access the first element - - Returns a reference to the first element in the container. For a JSON - container `c`, the expression `c.front()` is equivalent to `*c.begin()`. - - @return In case of a structured type (array or object), a reference to the - first element is returned. In cast of number, string, or boolean values, a - reference to the value is returned. - - @complexity Constant. - - @pre The JSON value must not be `null` (would throw `std::out_of_range`) - or an empty array or object (undefined behavior, guarded by assertions). - @post The JSON value remains unchanged. - - @throw std::out_of_range when called on `null` value - - @liveexample{The following code shows an example for `front()`.,front} - - @sa @ref back() -- access the last element - - @since version 1.0.0 - */ - reference front() - { - return *begin(); - } - - /*! - @copydoc basic_json::front() - */ - const_reference front() const - { - return *cbegin(); - } - - /*! - @brief access the last element - - Returns a reference to the last element in the container. For a JSON - container `c`, the expression `c.back()` is equivalent to - @code {.cpp} - auto tmp = c.end(); - --tmp; - return *tmp; - @endcode - - @return In case of a structured type (array or object), a reference to the - last element is returned. In cast of number, string, or boolean values, a - reference to the value is returned. - - @complexity Constant. - - @pre The JSON value must not be `null` (would throw `std::out_of_range`) - or an empty array or object (undefined behavior, guarded by assertions). - @post The JSON value remains unchanged. - - @throw std::out_of_range when called on `null` value. - - @liveexample{The following code shows an example for `back()`.,back} - - @sa @ref front() -- access the first element - - @since version 1.0.0 - */ - reference back() - { - auto tmp = end(); - --tmp; - return *tmp; - } - - /*! - @copydoc basic_json::back() - */ - const_reference back() const - { - auto tmp = cend(); - --tmp; - return *tmp; - } - - /*! - @brief remove element given an iterator - - Removes the element specified by iterator @a pos. The iterator @a pos must - be valid and dereferenceable. Thus the `end()` iterator (which is valid, - but is not dereferenceable) cannot be used as a value for @a pos. - - If called on a primitive type other than `null`, the resulting JSON value - will be `null`. - - @param[in] pos iterator to the element to remove - @return Iterator following the last removed element. If the iterator @a - pos refers to the last element, the `end()` iterator is returned. - - @tparam InteratorType an @ref iterator or @ref const_iterator - - @post Invalidates iterators and references at or after the point of the - erase, including the `end()` iterator. - - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on an iterator which does not belong to - the current JSON value; example: `"iterator does not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid - iterator (i.e., any iterator which is not `begin()`); example: `"iterator - out of range"` - - @complexity The complexity depends on the type: - - objects: amortized constant - - arrays: linear in distance between pos and the end of the container - - strings: linear in the length of the string - - other types: constant - - @liveexample{The example shows the result of `erase()` for different JSON - types.,erase__IteratorType} - - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - InteratorType erase(InteratorType pos) - { - // make sure iterator fits the current value - if (this != pos.m_object) - { - throw std::domain_error("iterator does not fit current value"); - } - - InteratorType result = end(); - - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not pos.m_it.primitive_iterator.is_begin()) - { - throw std::out_of_range("iterator out of range"); - } - - if (is_string()) - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - m_value.string = nullptr; - } - - m_type = value_t::null; - assert_invariant(); - break; - } - - case value_t::object: - { - result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); - break; - } - - case value_t::array: - { - result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - return result; - } - - /*! - @brief remove elements given an iterator range - - Removes the element specified by the range `[first; last)`. The iterator - @a first does not need to be dereferenceable if `first == last`: erasing - an empty range is a no-op. - - If called on a primitive type other than `null`, the resulting JSON value - will be `null`. - - @param[in] first iterator to the beginning of the range to remove - @param[in] last iterator past the end of the range to remove - @return Iterator following the last removed element. If the iterator @a - second refers to the last element, the `end()` iterator is returned. - - @tparam InteratorType an @ref iterator or @ref const_iterator - - @post Invalidates iterators and references at or after the point of the - erase, including the `end()` iterator. - - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on iterators which does not belong to - the current JSON value; example: `"iterators do not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid - iterators (i.e., if `first != begin()` and `last != end()`); example: - `"iterators out of range"` - - @complexity The complexity depends on the type: - - objects: `log(size()) + std::distance(first, last)` - - arrays: linear in the distance between @a first and @a last, plus linear - in the distance between @a last and end of the container - - strings: linear in the length of the string - - other types: constant - - @liveexample{The example shows the result of `erase()` for different JSON - types.,erase__IteratorType_IteratorType} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - template ::value or - std::is_same::value - , int>::type - = 0> - InteratorType erase(InteratorType first, InteratorType last) - { - // make sure iterator fits the current value - if (this != first.m_object or this != last.m_object) - { - throw std::domain_error("iterators do not fit current value"); - } - - InteratorType result = end(); - - switch (m_type) - { - case value_t::boolean: - case value_t::number_float: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::string: - { - if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) - { - throw std::out_of_range("iterators out of range"); - } - - if (is_string()) - { - AllocatorType alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); - m_value.string = nullptr; - } - - m_type = value_t::null; - assert_invariant(); - break; - } - - case value_t::object: - { - result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, - last.m_it.object_iterator); - break; - } - - case value_t::array: - { - result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, - last.m_it.array_iterator); - break; - } - - default: - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - return result; - } - - /*! - @brief remove element from a JSON object given a key - - Removes elements from a JSON object with the key value @a key. - - @param[in] key value of the elements to remove - - @return Number of elements removed. If @a ObjectType is the default - `std::map` type, the return value will always be `0` (@a key was not - found) or `1` (@a key was found). - - @post References and iterators to the erased elements are invalidated. - Other references and iterators are not affected. - - @throw std::domain_error when called on a type other than JSON object; - example: `"cannot use erase() with null"` - - @complexity `log(size()) + count(key)` - - @liveexample{The example shows the effect of `erase()`.,erase__key_type} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const size_type) -- removes the element from an array at - the given index - - @since version 1.0.0 - */ - size_type erase(const typename object_t::key_type& key) - { - // this erase only works for objects - if (is_object()) - { - return m_value.object->erase(key); - } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - /*! - @brief remove element from a JSON array given an index - - Removes element from a JSON array at the index @a idx. - - @param[in] idx index of the element to remove - - @throw std::domain_error when called on a type other than JSON array; - example: `"cannot use erase() with null"` - @throw std::out_of_range when `idx >= size()`; example: `"array index 17 - is out of range"` - - @complexity Linear in distance between @a idx and the end of the container. - - @liveexample{The example shows the effect of `erase()`.,erase__size_type} - - @sa @ref erase(InteratorType) -- removes the element at a given position - @sa @ref erase(InteratorType, InteratorType) -- removes the elements in - the given range - @sa @ref erase(const typename object_t::key_type&) -- removes the element - from an object at the given key - - @since version 1.0.0 - */ - void erase(const size_type idx) - { - // this erase only works for arrays - if (is_array()) - { - if (idx >= size()) - { - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - - m_value.array->erase(m_value.array->begin() + static_cast(idx)); - } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } - } - - /// @} - - - //////////// - // lookup // - //////////// - - /// @name lookup - /// @{ - - /*! - @brief find an element in a JSON object - - Finds an element in a JSON object with key equivalent to @a key. If the - element is not found or the JSON value is not an object, end() is - returned. - - @param[in] key key value of the element to search for - - @return Iterator to an element with key equivalent to @a key. If no such - element is found, past-the-end (see end()) iterator is returned. - - @complexity Logarithmic in the size of the JSON object. - - @liveexample{The example shows how `find()` is used.,find__key_type} - - @since version 1.0.0 - */ - iterator find(typename object_t::key_type key) - { - auto result = end(); - - if (is_object()) - { - result.m_it.object_iterator = m_value.object->find(key); - } - - return result; - } - - /*! - @brief find an element in a JSON object - @copydoc find(typename object_t::key_type) - */ - const_iterator find(typename object_t::key_type key) const - { - auto result = cend(); - - if (is_object()) - { - result.m_it.object_iterator = m_value.object->find(key); - } - - return result; - } - - /*! - @brief returns the number of occurrences of a key in a JSON object - - Returns the number of elements with key @a key. If ObjectType is the - default `std::map` type, the return value will always be `0` (@a key was - not found) or `1` (@a key was found). - - @param[in] key key value of the element to count - - @return Number of elements with key @a key. If the JSON value is not an - object, the return value will be `0`. - - @complexity Logarithmic in the size of the JSON object. - - @liveexample{The example shows how `count()` is used.,count} - - @since version 1.0.0 - */ - size_type count(typename object_t::key_type key) const - { - // return 0 for all nonobject types - return is_object() ? m_value.object->count(key) : 0; - } - - /// @} - - - /////////////// - // iterators // - /////////////// - - /// @name iterators - /// @{ - - /*! - @brief returns an iterator to the first element - - Returns an iterator to the first element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return iterator to the first element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - @liveexample{The following code shows an example for `begin()`.,begin} - - @sa @ref cbegin() -- returns a const iterator to the beginning - @sa @ref end() -- returns an iterator to the end - @sa @ref cend() -- returns a const iterator to the end - - @since version 1.0.0 - */ - iterator begin() noexcept - { - iterator result(this); - result.set_begin(); - return result; - } - - /*! - @copydoc basic_json::cbegin() - */ - const_iterator begin() const noexcept - { - return cbegin(); - } - - /*! - @brief returns a const iterator to the first element - - Returns a const iterator to the first element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return const iterator to the first element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).begin()`. - - @liveexample{The following code shows an example for `cbegin()`.,cbegin} - - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref end() -- returns an iterator to the end - @sa @ref cend() -- returns a const iterator to the end - - @since version 1.0.0 - */ - const_iterator cbegin() const noexcept - { - const_iterator result(this); - result.set_begin(); - return result; - } - - /*! - @brief returns an iterator to one past the last element - - Returns an iterator to one past the last element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return iterator one past the last element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - @liveexample{The following code shows an example for `end()`.,end} - - @sa @ref cend() -- returns a const iterator to the end - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref cbegin() -- returns a const iterator to the beginning - - @since version 1.0.0 - */ - iterator end() noexcept - { - iterator result(this); - result.set_end(); - return result; - } - - /*! - @copydoc basic_json::cend() - */ - const_iterator end() const noexcept - { - return cend(); - } - - /*! - @brief returns a const iterator to one past the last element - - Returns a const iterator to one past the last element. - - @image html range-begin-end.svg "Illustration from cppreference.com" - - @return const iterator one past the last element - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).end()`. - - @liveexample{The following code shows an example for `cend()`.,cend} - - @sa @ref end() -- returns an iterator to the end - @sa @ref begin() -- returns an iterator to the beginning - @sa @ref cbegin() -- returns a const iterator to the beginning - - @since version 1.0.0 - */ - const_iterator cend() const noexcept - { - const_iterator result(this); - result.set_end(); - return result; - } - - /*! - @brief returns an iterator to the reverse-beginning - - Returns an iterator to the reverse-beginning; that is, the last element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `reverse_iterator(end())`. - - @liveexample{The following code shows an example for `rbegin()`.,rbegin} - - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref crend() -- returns a const reverse iterator to the end - - @since version 1.0.0 - */ - reverse_iterator rbegin() noexcept - { - return reverse_iterator(end()); - } - - /*! - @copydoc basic_json::crbegin() - */ - const_reverse_iterator rbegin() const noexcept - { - return crbegin(); - } - - /*! - @brief returns an iterator to the reverse-end - - Returns an iterator to the reverse-end; that is, one before the first - element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `reverse_iterator(begin())`. - - @liveexample{The following code shows an example for `rend()`.,rend} - - @sa @ref crend() -- returns a const reverse iterator to the end - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - - @since version 1.0.0 - */ - reverse_iterator rend() noexcept - { - return reverse_iterator(begin()); - } - - /*! - @copydoc basic_json::crend() - */ - const_reverse_iterator rend() const noexcept - { - return crend(); - } - - /*! - @brief returns a const reverse iterator to the last element - - Returns a const iterator to the reverse-beginning; that is, the last - element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).rbegin()`. - - @liveexample{The following code shows an example for `crbegin()`.,crbegin} - - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref crend() -- returns a const reverse iterator to the end - - @since version 1.0.0 - */ - const_reverse_iterator crbegin() const noexcept - { - return const_reverse_iterator(cend()); - } - - /*! - @brief returns a const reverse iterator to one before the first - - Returns a const reverse iterator to the reverse-end; that is, one before - the first element. - - @image html range-rbegin-rend.svg "Illustration from cppreference.com" - - @complexity Constant. - - @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) - requirements: - - The complexity is constant. - - Has the semantics of `const_cast(*this).rend()`. - - @liveexample{The following code shows an example for `crend()`.,crend} - - @sa @ref rend() -- returns a reverse iterator to the end - @sa @ref rbegin() -- returns a reverse iterator to the beginning - @sa @ref crbegin() -- returns a const reverse iterator to the beginning - - @since version 1.0.0 - */ - const_reverse_iterator crend() const noexcept - { - return const_reverse_iterator(cbegin()); - } - - private: - // forward declaration - template class iteration_proxy; - - public: - /*! - @brief wrapper to access iterator member functions in range-based for - - This function allows to access @ref iterator::key() and @ref - iterator::value() during range-based for loops. In these loops, a - reference to the JSON values is returned, so there is no access to the - underlying iterator. - - @note The name of this function is not yet final and may change in the - future. - */ - static iteration_proxy iterator_wrapper(reference cont) - { - return iteration_proxy(cont); - } - - /*! - @copydoc iterator_wrapper(reference) - */ - static iteration_proxy iterator_wrapper(const_reference cont) - { - return iteration_proxy(cont); - } - - /// @} - - - ////////////// - // capacity // - ////////////// - - /// @name capacity - /// @{ - - /*! - @brief checks whether the container is empty - - Checks if a JSON value has no elements. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `true` - boolean | `false` - string | `false` - number | `false` - object | result of function `object_t::empty()` - array | result of function `array_t::empty()` - - @note This function does not return whether a string stored as JSON value - is empty - it returns whether the JSON container itself is empty which is - false in the case of a string. - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their `empty()` functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `begin() == end()`. - - @liveexample{The following code uses `empty()` to check if a JSON - object contains any elements.,empty} - - @sa @ref size() -- returns the number of elements - - @since version 1.0.0 - */ - bool empty() const noexcept - { - switch (m_type) - { - case value_t::null: - { - // null values are empty - return true; - } - - case value_t::array: - { - // delegate call to array_t::empty() - return m_value.array->empty(); - } - - case value_t::object: - { - // delegate call to object_t::empty() - return m_value.object->empty(); - } - - default: - { - // all other types are nonempty - return false; - } - } - } - - /*! - @brief returns the number of elements - - Returns the number of elements in a JSON value. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `0` - boolean | `1` - string | `1` - number | `1` - object | result of function object_t::size() - array | result of function array_t::size() - - @note This function does not return the length of a string stored as JSON - value - it returns the number of elements in the JSON value which is 1 in - the case of a string. - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their size() functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of `std::distance(begin(), end())`. - - @liveexample{The following code calls `size()` on the different value - types.,size} - - @sa @ref empty() -- checks whether the container is empty - @sa @ref max_size() -- returns the maximal number of elements - - @since version 1.0.0 - */ - size_type size() const noexcept - { - switch (m_type) - { - case value_t::null: - { - // null values are empty - return 0; - } - - case value_t::array: - { - // delegate call to array_t::size() - return m_value.array->size(); - } - - case value_t::object: - { - // delegate call to object_t::size() - return m_value.object->size(); - } - - default: - { - // all other types have size 1 - return 1; - } - } - } - - /*! - @brief returns the maximum possible number of elements - - Returns the maximum number of elements a JSON value is able to hold due to - system or library implementation limitations, i.e. `std::distance(begin(), - end())` for the JSON value. - - @return The return value depends on the different types and is - defined as follows: - Value type | return value - ----------- | ------------- - null | `0` (same as `size()`) - boolean | `1` (same as `size()`) - string | `1` (same as `size()`) - number | `1` (same as `size()`) - object | result of function `object_t::max_size()` - array | result of function `array_t::max_size()` - - @complexity Constant, as long as @ref array_t and @ref object_t satisfy - the Container concept; that is, their `max_size()` functions have constant - complexity. - - @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) - requirements: - - The complexity is constant. - - Has the semantics of returning `b.size()` where `b` is the largest - possible JSON value. - - @liveexample{The following code calls `max_size()` on the different value - types. Note the output is implementation specific.,max_size} - - @sa @ref size() -- returns the number of elements - - @since version 1.0.0 - */ - size_type max_size() const noexcept - { - switch (m_type) - { - case value_t::array: - { - // delegate call to array_t::max_size() - return m_value.array->max_size(); - } - - case value_t::object: - { - // delegate call to object_t::max_size() - return m_value.object->max_size(); - } - - default: - { - // all other types have max_size() == size() - return size(); - } - } - } - - /// @} - - - /////////////// - // modifiers // - /////////////// - - /// @name modifiers - /// @{ - - /*! - @brief clears the contents - - Clears the content of a JSON value and resets it to the default value as - if @ref basic_json(value_t) would have been called: - - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` - - @note Floating-point numbers are set to `0.0` which will be serialized to - `0`. The vale type remains @ref number_float_t. - - @complexity Linear in the size of the JSON value. - - @liveexample{The example below shows the effect of `clear()` to different - JSON types.,clear} - - @since version 1.0.0 - */ - void clear() noexcept - { - switch (m_type) - { - case value_t::number_integer: - { - m_value.number_integer = 0; - break; - } - - case value_t::number_unsigned: - { - m_value.number_unsigned = 0; - break; - } - - case value_t::number_float: - { - m_value.number_float = 0.0; - break; - } - - case value_t::boolean: - { - m_value.boolean = false; - break; - } - - case value_t::string: - { - m_value.string->clear(); - break; - } - - case value_t::array: - { - m_value.array->clear(); - break; - } - - case value_t::object: - { - m_value.object->clear(); - break; - } - - default: - { - break; - } - } - } - - /*! - @brief add an object to an array - - Appends the given element @a val to the end of the JSON value. If the - function is called on a JSON null value, an empty array is created before - appending @a val. - - @param[in] val the value to add to the JSON array - - @throw std::domain_error when called on a type other than JSON array or - null; example: `"cannot use push_back() with number"` - - @complexity Amortized constant. - - @liveexample{The example shows how `push_back()` and `+=` can be used to - add elements to a JSON array. Note how the `null` value was silently - converted to a JSON array.,push_back} - - @since version 1.0.0 - */ - void push_back(basic_json&& val) - { - // push_back only works for null objects or arrays - if (not(is_null() or is_array())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an array - if (is_null()) - { - m_type = value_t::array; - m_value = value_t::array; - assert_invariant(); - } - - // add element to array (move semantics) - m_value.array->push_back(std::move(val)); - // invalidate object - val.m_type = value_t::null; - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - reference operator+=(basic_json&& val) - { - push_back(std::move(val)); - return *this; - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - void push_back(const basic_json& val) - { - // push_back only works for null objects or arrays - if (not(is_null() or is_array())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an array - if (is_null()) - { - m_type = value_t::array; - m_value = value_t::array; - assert_invariant(); - } - - // add element to array - m_value.array->push_back(val); - } - - /*! - @brief add an object to an array - @copydoc push_back(basic_json&&) - */ - reference operator+=(const basic_json& val) - { - push_back(val); - return *this; - } - - /*! - @brief add an object to an object - - Inserts the given element @a val to the JSON object. If the function is - called on a JSON null value, an empty object is created before inserting - @a val. - - @param[in] val the value to add to the JSON object - - @throw std::domain_error when called on a type other than JSON object or - null; example: `"cannot use push_back() with number"` - - @complexity Logarithmic in the size of the container, O(log(`size()`)). - - @liveexample{The example shows how `push_back()` and `+=` can be used to - add elements to a JSON object. Note how the `null` value was silently - converted to a JSON object.,push_back__object_t__value} - - @since version 1.0.0 - */ - void push_back(const typename object_t::value_type& val) - { - // push_back only works for null objects or objects - if (not(is_null() or is_object())) - { - throw std::domain_error("cannot use push_back() with " + type_name()); - } - - // transform null object into an object - if (is_null()) - { - m_type = value_t::object; - m_value = value_t::object; - assert_invariant(); - } - - // add element to array - m_value.object->insert(val); - } - - /*! - @brief add an object to an object - @copydoc push_back(const typename object_t::value_type&) - */ - reference operator+=(const typename object_t::value_type& val) - { - push_back(val); - return *this; - } - - /*! - @brief add an object to an object - - This function allows to use `push_back` with an initializer list. In case - - 1. the current value is an object, - 2. the initializer list @a init contains only two elements, and - 3. the first element of @a init is a string, - - @a init is converted into an object element and added using - @ref push_back(const typename object_t::value_type&). Otherwise, @a init - is converted to a JSON value and added using @ref push_back(basic_json&&). - - @param init an initializer list - - @complexity Linear in the size of the initializer list @a init. - - @note This function is required to resolve an ambiguous overload error, - because pairs like `{"key", "value"}` can be both interpreted as - `object_t::value_type` or `std::initializer_list`, see - https://github.com/nlohmann/json/issues/235 for more information. - - @liveexample{The example shows how initializer lists are treated as - objects when possible.,push_back__initializer_list} - */ - void push_back(std::initializer_list init) - { - if (is_object() and init.size() == 2 and init.begin()->is_string()) - { - const string_t key = *init.begin(); - push_back(typename object_t::value_type(key, *(init.begin() + 1))); - } - else - { - push_back(basic_json(init)); - } - } - - /*! - @brief add an object to an object - @copydoc push_back(std::initializer_list) - */ - reference operator+=(std::initializer_list init) - { - push_back(init); - return *this; - } - - /*! - @brief inserts element - - Inserts element @a val before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] val element to insert - @return iterator pointing to the inserted @a val. - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @complexity Constant plus linear in the distance between pos and end of the - container. - - @liveexample{The example shows how `insert()` is used.,insert} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, const basic_json& val) - { - // insert only works for arrays - if (is_array()) - { - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); - return result; - } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - } - - /*! - @brief inserts element - @copydoc insert(const_iterator, const basic_json&) - */ - iterator insert(const_iterator pos, basic_json&& val) - { - return insert(pos, val); - } - - /*! - @brief inserts elements - - Inserts @a cnt copies of @a val before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] cnt number of copies of @a val to insert - @param[in] val element to insert - @return iterator pointing to the first element inserted, or @a pos if - `cnt==0` - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @complexity Linear in @a cnt plus linear in the distance between @a pos - and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__count} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, size_type cnt, const basic_json& val) - { - // insert only works for arrays - if (is_array()) - { - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); - return result; - } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - } - - /*! - @brief inserts elements - - Inserts elements from range `[first, last)` before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] first begin of the range of elements to insert - @param[in] last end of the range of elements to insert - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - @throw std::domain_error if @a first and @a last do not belong to the same - JSON value; example: `"iterators do not fit"` - @throw std::domain_error if @a first or @a last are iterators into - container for which insert is called; example: `"passed iterators may not - belong to container"` - - @return iterator pointing to the first element inserted, or @a pos if - `first==last` - - @complexity Linear in `std::distance(first, last)` plus linear in the - distance between @a pos and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__range} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, const_iterator first, const_iterator last) - { - // insert only works for arrays - if (not is_array()) - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // check if range iterators belong to the same JSON object - if (first.m_object != last.m_object) - { - throw std::domain_error("iterators do not fit"); - } - - if (first.m_object == this or last.m_object == this) - { - throw std::domain_error("passed iterators may not belong to container"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert( - pos.m_it.array_iterator, - first.m_it.array_iterator, - last.m_it.array_iterator); - return result; - } - - /*! - @brief inserts elements - - Inserts elements from initializer list @a ilist before iterator @a pos. - - @param[in] pos iterator before which the content will be inserted; may be - the end() iterator - @param[in] ilist initializer list to insert the values from - - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - - @return iterator pointing to the first element inserted, or @a pos if - `ilist` is empty - - @complexity Linear in `ilist.size()` plus linear in the distance between - @a pos and end of the container. - - @liveexample{The example shows how `insert()` is used.,insert__ilist} - - @since version 1.0.0 - */ - iterator insert(const_iterator pos, std::initializer_list ilist) - { - // insert only works for arrays - if (not is_array()) - { - throw std::domain_error("cannot use insert() with " + type_name()); - } - - // check if iterator pos fits to this JSON value - if (pos.m_object != this) - { - throw std::domain_error("iterator does not fit current value"); - } - - // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); - return result; - } - - /*! - @brief exchanges the values - - Exchanges the contents of the JSON value with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other JSON value to exchange the contents with - - @complexity Constant. - - @liveexample{The example below shows how JSON values can be swapped with - `swap()`.,swap__reference} - - @since version 1.0.0 - */ - void swap(reference other) noexcept ( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - std::swap(m_type, other.m_type); - std::swap(m_value, other.m_value); - assert_invariant(); - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON array with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other array to exchange the contents with - - @throw std::domain_error when JSON value is not an array; example: `"cannot - use swap() with string"` - - @complexity Constant. - - @liveexample{The example below shows how arrays can be swapped with - `swap()`.,swap__array_t} - - @since version 1.0.0 - */ - void swap(array_t& other) - { - // swap only works for arrays - if (is_array()) - { - std::swap(*(m_value.array), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON object with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other object to exchange the contents with - - @throw std::domain_error when JSON value is not an object; example: - `"cannot use swap() with string"` - - @complexity Constant. - - @liveexample{The example below shows how objects can be swapped with - `swap()`.,swap__object_t} - - @since version 1.0.0 - */ - void swap(object_t& other) - { - // swap only works for objects - if (is_object()) - { - std::swap(*(m_value.object), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /*! - @brief exchanges the values - - Exchanges the contents of a JSON string with those of @a other. Does not - invoke any move, copy, or swap operations on individual elements. All - iterators and references remain valid. The past-the-end iterator is - invalidated. - - @param[in,out] other string to exchange the contents with - - @throw std::domain_error when JSON value is not a string; example: `"cannot - use swap() with boolean"` - - @complexity Constant. - - @liveexample{The example below shows how strings can be swapped with - `swap()`.,swap__string_t} - - @since version 1.0.0 - */ - void swap(string_t& other) - { - // swap only works for strings - if (is_string()) - { - std::swap(*(m_value.string), other); - } - else - { - throw std::domain_error("cannot use swap() with " + type_name()); - } - } - - /// @} - - - ////////////////////////////////////////// - // lexicographical comparison operators // - ////////////////////////////////////////// - - /// @name lexicographical comparison operators - /// @{ - - private: - /*! - @brief comparison operator for JSON types - - Returns an ordering that is similar to Python: - - order: null < boolean < number < object < array < string - - furthermore, each type is not smaller than itself - - @since version 1.0.0 - */ - friend bool operator<(const value_t lhs, const value_t rhs) noexcept - { - static constexpr std::array order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float - } - }; - - // discarded values are not comparable - if (lhs == value_t::discarded or rhs == value_t::discarded) - { - return false; - } - - return order[static_cast(lhs)] < order[static_cast(rhs)]; - } - - public: - /*! - @brief comparison: equal - - Compares two JSON values for equality according to the following rules: - - Two JSON values are equal if (1) they are from the same type and (2) - their stored values are the same. - - Integer and floating-point numbers are automatically converted before - comparison. Floating-point numbers are compared indirectly: two - floating-point numbers `f1` and `f2` are considered equal if neither - `f1 > f2` nor `f2 > f1` holds. - - Two JSON null values are equal. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are equal - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__equal} - - @since version 1.0.0 - */ - friend bool operator==(const_reference lhs, const_reference rhs) noexcept - { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); - - if (lhs_type == rhs_type) - { - switch (lhs_type) - { - case value_t::array: - { - return *lhs.m_value.array == *rhs.m_value.array; - } - case value_t::object: - { - return *lhs.m_value.object == *rhs.m_value.object; - } - case value_t::null: - { - return true; - } - case value_t::string: - { - return *lhs.m_value.string == *rhs.m_value.string; - } - case value_t::boolean: - { - return lhs.m_value.boolean == rhs.m_value.boolean; - } - case value_t::number_integer: - { - return lhs.m_value.number_integer == rhs.m_value.number_integer; - } - case value_t::number_unsigned: - { - return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; - } - case value_t::number_float: - { - return lhs.m_value.number_float == rhs.m_value.number_float; - } - default: - { - return false; - } - } - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) - { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); - } - - return false; - } - - /*! - @brief comparison: equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__equal__nullptr_t} - - @since version 1.0.0 - */ - friend bool operator==(const_reference v, std::nullptr_t) noexcept - { - return v.is_null(); - } - - /*! - @brief comparison: equal - @copydoc operator==(const_reference, std::nullptr_t) - */ - friend bool operator==(std::nullptr_t, const_reference v) noexcept - { - return v.is_null(); - } - - /*! - @brief comparison: not equal - - Compares two JSON values for inequality by calculating `not (lhs == rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are not equal - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__notequal} - - @since version 1.0.0 - */ - friend bool operator!=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs == rhs); - } - - /*! - @brief comparison: not equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `not v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is not null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__notequal__nullptr_t} - - @since version 1.0.0 - */ - friend bool operator!=(const_reference v, std::nullptr_t) noexcept - { - return not v.is_null(); - } - - /*! - @brief comparison: not equal - @copydoc operator!=(const_reference, std::nullptr_t) - */ - friend bool operator!=(std::nullptr_t, const_reference v) noexcept - { - return not v.is_null(); - } - - /*! - @brief comparison: less than - - Compares whether one JSON value @a lhs is less than another JSON value @a - rhs according to the following rules: - - If @a lhs and @a rhs have the same type, the values are compared using - the default `<` operator. - - Integer and floating-point numbers are automatically converted before - comparison - - In case @a lhs and @a rhs have different types, the values are ignored - and the order of the types is considered, see - @ref operator<(const value_t, const value_t). - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__less} - - @since version 1.0.0 - */ - friend bool operator<(const_reference lhs, const_reference rhs) noexcept - { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); - - if (lhs_type == rhs_type) - { - switch (lhs_type) - { - case value_t::array: - { - return *lhs.m_value.array < *rhs.m_value.array; - } - case value_t::object: - { - return *lhs.m_value.object < *rhs.m_value.object; - } - case value_t::null: - { - return false; - } - case value_t::string: - { - return *lhs.m_value.string < *rhs.m_value.string; - } - case value_t::boolean: - { - return lhs.m_value.boolean < rhs.m_value.boolean; - } - case value_t::number_integer: - { - return lhs.m_value.number_integer < rhs.m_value.number_integer; - } - case value_t::number_unsigned: - { - return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; - } - case value_t::number_float: - { - return lhs.m_value.number_float < rhs.m_value.number_float; - } - default: - { - return false; - } - } - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; - } - - // We only reach this line if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - return operator<(lhs_type, rhs_type); - } - - /*! - @brief comparison: less than or equal - - Compares whether one JSON value @a lhs is less than or equal to another - JSON value by calculating `not (rhs < lhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than or equal to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__greater} - - @since version 1.0.0 - */ - friend bool operator<=(const_reference lhs, const_reference rhs) noexcept - { - return not (rhs < lhs); - } - - /*! - @brief comparison: greater than - - Compares whether one JSON value @a lhs is greater than another - JSON value by calculating `not (lhs <= rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__lessequal} - - @since version 1.0.0 - */ - friend bool operator>(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs <= rhs); - } - - /*! - @brief comparison: greater than or equal - - Compares whether one JSON value @a lhs is greater than or equal to another - JSON value by calculating `not (lhs < rhs)`. - - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than or equal to @a rhs - - @complexity Linear. - - @liveexample{The example demonstrates comparing several JSON - types.,operator__greaterequal} - - @since version 1.0.0 - */ - friend bool operator>=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs < rhs); - } - - /// @} - - - /////////////////// - // serialization // - /////////////////// - - /// @name serialization - /// @{ - - /*! - @brief serialize to stream - - Serialize the given JSON value @a j to the output stream @a o. The JSON - value will be serialized using the @ref dump member function. The - indentation of the output can be controlled with the member variable - `width` of the output stream @a o. For instance, using the manipulator - `std::setw(4)` on @a o sets the indentation level to `4` and the - serialization result is the same as calling `dump(4)`. - - @note During serializaion, the locale and the precision of the output - stream @a o are changed. The original values are restored when the - function returns. - - @param[in,out] o stream to serialize to - @param[in] j JSON value to serialize - - @return the stream @a o - - @complexity Linear. - - @liveexample{The example below shows the serialization with different - parameters to `width` to adjust the indentation level.,operator_serialize} - - @since version 1.0.0 - */ - friend std::ostream& operator<<(std::ostream& o, const basic_json& j) - { - // read width member and use it as indentation parameter if nonzero - const bool pretty_print = (o.width() > 0); - const auto indentation = (pretty_print ? o.width() : 0); - - // reset width to 0 for subsequent calls to this stream - o.width(0); - - // fix locale problems - const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator)); - // set precision - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - const auto old_precision = o.precision(std::numeric_limits::digits10); - - // do the actual serialization - j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale and precision - o.imbue(old_locale); - o.precision(old_precision); - return o; - } - - /*! - @brief serialize to stream - @copydoc operator<<(std::ostream&, const basic_json&) - */ - friend std::ostream& operator>>(const basic_json& j, std::ostream& o) - { - return o << j; - } - - /// @} - - - ///////////////////// - // deserialization // - ///////////////////// - - /// @name deserialization - /// @{ - - /*! - @brief deserialize from string - - @param[in] s string to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__string__parser_callback_t} - - @sa @ref parse(std::istream&, const parser_callback_t) for a version that - reads from an input stream - - @since version 1.0.0 - */ - static basic_json parse(const string_t& s, - const parser_callback_t cb = nullptr) - { - return parser(s, cb).parse(); - } - - /*! - @brief deserialize from stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @return result of the deserialization - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below demonstrates the `parse()` function with - and without callback function.,parse__istream__parser_callback_t} - - @sa @ref parse(const string_t&, const parser_callback_t) for a version - that reads from a string - - @since version 1.0.0 - */ - static basic_json parse(std::istream& i, - const parser_callback_t cb = nullptr) - { - return parser(i, cb).parse(); - } - - /*! - @copydoc parse(std::istream&, const parser_callback_t) - */ - static basic_json parse(std::istream&& i, - const parser_callback_t cb = nullptr) - { - return parser(i, cb).parse(); - } - - /*! - @brief deserialize from stream - - Deserializes an input stream to a JSON value. - - @param[in,out] i input stream to read a serialized JSON value from - @param[in,out] j JSON value to write the deserialized input to - - @throw std::invalid_argument in case of parse errors - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. - - @note A UTF-8 byte order mark is silently ignored. - - @liveexample{The example below shows how a JSON value is constructed by - reading a serialization from a stream.,operator_deserialize} - - @sa parse(std::istream&, const parser_callback_t) for a variant with a - parser callback function to filter values while parsing - - @since version 1.0.0 - */ - friend std::istream& operator<<(basic_json& j, std::istream& i) - { - j = parser(i).parse(); - return i; - } - - /*! - @brief deserialize from stream - @copydoc operator<<(basic_json&, std::istream&) - */ - friend std::istream& operator>>(std::istream& i, basic_json& j) - { - j = parser(i).parse(); - return i; - } - - /// @} - - - private: - /////////////////////////// - // convenience functions // - /////////////////////////// - - /*! - @brief return the type as string - - Returns the type name as string to be used in error messages - usually to - indicate that a function was called on a wrong JSON type. - - @return basically a string representation of a the @ref m_type member - - @complexity Constant. - - @since version 1.0.0 - */ - std::string type_name() const - { - switch (m_type) - { - case value_t::null: - return "null"; - case value_t::object: - return "object"; - case value_t::array: - return "array"; - case value_t::string: - return "string"; - case value_t::boolean: - return "boolean"; - case value_t::discarded: - return "discarded"; - default: - return "number"; - } - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s) noexcept - { - return std::accumulate(s.begin(), s.end(), size_t{}, - [](size_t res, typename string_t::value_type c) - { - switch (c) - { - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - return res + 1; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // from c (1 byte) to \uxxxx (6 bytes) - return res + 5; - } - else - { - return res; - } - } - } - }); - } - - /*! - @brief escape a string - - Escape a string by replacing certain special characters by a sequence of - an escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. - - @param[in] s the string to escape - @return the escaped string - - @complexity Linear in the length of string @a s. - */ - static string_t escape_string(const string_t& s) - { - const auto space = extra_space(s); - if (space == 0) - { - return s; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; - - for (const auto& c : s) - { - switch (c) - { - // quotation mark (0x22) - case '"': - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - // reverse solidus (0x5c) - case '\\': - { - // nothing to change - pos += 2; - break; - } - - // backspace (0x08) - case '\b': - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - // formfeed (0x0c) - case '\f': - { - result[pos + 1] = 'f'; - pos += 2; - break; - } - - // newline (0x0a) - case '\n': - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - // carriage return (0x0d) - case '\r': - { - result[pos + 1] = 'r'; - pos += 2; - break; - } - - // horizontal tab (0x09) - case '\t': - { - result[pos + 1] = 't'; - pos += 2; - break; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // convert a number 0..15 to its hex representation - // (0..f) - static const char hexify[16] = - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - - // print character c as \uxxxx - for (const char m : - { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] - }) - { - result[++pos] = m; - } - - ++pos; - } - else - { - // all other characters are added as-is - result[pos++] = c; - } - break; - } - } - } - - return result; - } - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. Note that - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[out] o stream to write to - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(std::ostream& o, - const bool pretty_print, - const unsigned int indent_step, - const unsigned int current_indent = 0) const - { - // variable to hold indentation for recursive calls - unsigned int new_indent = current_indent; - - switch (m_type) - { - case value_t::object: - { - if (m_value.object->empty()) - { - o << "{}"; - return; - } - - o << "{"; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) - { - if (i != m_value.object->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' ') << "\"" - << escape_string(i->first) << "\":" - << (pretty_print ? " " : ""); - i->second.dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') + "}"; - return; - } - - case value_t::array: - { - if (m_value.array->empty()) - { - o << "[]"; - return; - } - - o << "["; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) - { - if (i != m_value.array->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' '); - i->dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') << "]"; - return; - } - - case value_t::string: - { - o << string_t("\"") << escape_string(*m_value.string) << "\""; - return; - } - - case value_t::boolean: - { - o << (m_value.boolean ? "true" : "false"); - return; - } - - case value_t::number_integer: - { - o << m_value.number_integer; - return; - } - - case value_t::number_unsigned: - { - o << m_value.number_unsigned; - return; - } - - case value_t::number_float: - { - if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); - } - else - { - o << m_value.number_float; - } - return; - } - - case value_t::discarded: - { - o << ""; - return; - } - - case value_t::null: - { - o << "null"; - return; - } - } - } - - private: - ////////////////////// - // member variables // - ////////////////////// - - /// the type of the current element - value_t m_type = value_t::null; - - /// the value of the current element - json_value m_value = {}; - - - private: - /////////////// - // iterators // - /////////////// - - /*! - @brief an iterator for primitive JSON types - - This class models an iterator for primitive JSON types (boolean, number, - string). It's only purpose is to allow the iterator/const_iterator classes - to "iterate" over primitive values. Internally, the iterator is modeled by - a `difference_type` variable. Value begin_value (`0`) models the begin, - end_value (`1`) models past the end. - */ - class primitive_iterator_t - { - public: - /// set iterator to a defined beginning - void set_begin() noexcept - { - m_it = begin_value; - } - - /// set iterator to a defined past the end - void set_end() noexcept - { - m_it = end_value; - } - - /// return whether the iterator can be dereferenced - constexpr bool is_begin() const noexcept - { - return (m_it == begin_value); - } - - /// return whether the iterator is at end - constexpr bool is_end() const noexcept - { - return (m_it == end_value); - } - - /// return reference to the value to change and compare - operator difference_type& () noexcept - { - return m_it; - } - - /// return value to compare - constexpr operator difference_type () const noexcept - { - return m_it; - } - - private: - static constexpr difference_type begin_value = 0; - static constexpr difference_type end_value = begin_value + 1; - - /// iterator as signed integer type - difference_type m_it = std::numeric_limits::denorm_min(); - }; - - /*! - @brief an iterator value - - @note This structure could easily be a union, but MSVC currently does not - allow unions members with complex constructors, see - https://github.com/nlohmann/json/pull/105. - */ - struct internal_iterator - { - /// iterator for JSON objects - typename object_t::iterator object_iterator; - /// iterator for JSON arrays - typename array_t::iterator array_iterator; - /// generic iterator for all other types - primitive_iterator_t primitive_iterator; - - /// create an uninitialized internal_iterator - internal_iterator() noexcept - : object_iterator(), array_iterator(), primitive_iterator() - {} - }; - - /// proxy class for the iterator_wrapper functions - template - class iteration_proxy - { - private: - /// helper class for iteration - class iteration_proxy_internal - { - private: - /// the iterator - IteratorType anchor; - /// an index for arrays (used to create key names) - size_t array_index = 0; - - public: - explicit iteration_proxy_internal(IteratorType it) noexcept - : anchor(it) - {} - - /// dereference operator (needed for range-based for) - iteration_proxy_internal& operator*() - { - return *this; - } - - /// increment operator (needed for range-based for) - iteration_proxy_internal& operator++() - { - ++anchor; - ++array_index; - - return *this; - } - - /// inequality operator (needed for range-based for) - bool operator!= (const iteration_proxy_internal& o) const - { - return anchor != o.anchor; - } - - /// return key of the iterator - typename basic_json::string_t key() const - { - assert(anchor.m_object != nullptr); - - switch (anchor.m_object->type()) - { - // use integer array index as key - case value_t::array: - { - return std::to_string(array_index); - } - - // use key from the object - case value_t::object: - { - return anchor.key(); - } - - // use an empty key for all primitive types - default: - { - return ""; - } - } - } - - /// return value of the iterator - typename IteratorType::reference value() const - { - return anchor.value(); - } - }; - - /// the container to iterate - typename IteratorType::reference container; - - public: - /// construct iteration proxy from a container - explicit iteration_proxy(typename IteratorType::reference cont) - : container(cont) - {} - - /// return iterator begin (needed for range-based for) - iteration_proxy_internal begin() noexcept - { - return iteration_proxy_internal(container.begin()); - } - - /// return iterator end (needed for range-based for) - iteration_proxy_internal end() noexcept - { - return iteration_proxy_internal(container.end()); - } - }; - - public: - /*! - @brief a const random access iterator for the @ref basic_json class - - This class implements a const iterator for the @ref basic_json class. From - this class, the @ref iterator class is derived. - - @note An iterator is called *initialized* when a pointer to a JSON value - has been set (e.g., by a constructor or a copy assignment). If the - iterator is default-constructed, it is *uninitialized* and most - methods are undefined. The library uses assertions to detect calls - on uninitialized iterators. - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - @since version 1.0.0 - */ - class const_iterator : public std::iterator - { - /// allow basic_json to access private members - friend class basic_json; - - public: - /// the type of the values when the iterator is dereferenced - using value_type = typename basic_json::value_type; - /// a type to represent differences between iterators - using difference_type = typename basic_json::difference_type; - /// defines a pointer to the type iterated over (value_type) - using pointer = typename basic_json::const_pointer; - /// defines a reference to the type iterated over (value_type) - using reference = typename basic_json::const_reference; - /// the category of the iterator - using iterator_category = std::bidirectional_iterator_tag; - - /// default constructor - const_iterator() = default; - - /*! - @brief constructor for a given JSON instance - @param[in] object pointer to a JSON object for this iterator - @pre object != nullptr - @post The iterator is initialized; i.e. `m_object != nullptr`. - */ - explicit const_iterator(pointer object) noexcept - : m_object(object) - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = typename object_t::iterator(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = typename array_t::iterator(); - break; - } - - default: - { - m_it.primitive_iterator = primitive_iterator_t(); - break; - } - } - } - - /*! - @brief copy constructor given a non-const iterator - @param[in] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - explicit const_iterator(const iterator& other) noexcept - : m_object(other.m_object) - { - if (m_object != nullptr) - { - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = other.m_it.object_iterator; - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = other.m_it.array_iterator; - break; - } - - default: - { - m_it.primitive_iterator = other.m_it.primitive_iterator; - break; - } - } - } - } - - /*! - @brief copy constructor - @param[in] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - const_iterator(const const_iterator& other) noexcept - : m_object(other.m_object), m_it(other.m_it) - {} - - /*! - @brief copy assignment - @param[in,out] other iterator to copy from - @note It is not checked whether @a other is initialized. - */ - const_iterator& operator=(const_iterator other) noexcept( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - std::swap(m_object, other.m_object); - std::swap(m_it, other.m_it); - return *this; - } - - private: - /*! - @brief set the iterator to the first value - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - void set_begin() noexcept - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = m_object->m_value.object->begin(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = m_object->m_value.array->begin(); - break; - } - - case basic_json::value_t::null: - { - // set to end so begin()==end() is true: null is empty - m_it.primitive_iterator.set_end(); - break; - } - - default: - { - m_it.primitive_iterator.set_begin(); - break; - } - } - } - - /*! - @brief set the iterator past the last value - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - void set_end() noexcept - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - m_it.object_iterator = m_object->m_value.object->end(); - break; - } - - case basic_json::value_t::array: - { - m_it.array_iterator = m_object->m_value.array->end(); - break; - } - - default: - { - m_it.primitive_iterator.set_end(); - break; - } - } - } - - public: - /*! - @brief return a reference to the value pointed to by the iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference operator*() const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - assert(m_it.object_iterator != m_object->m_value.object->end()); - return m_it.object_iterator->second; - } - - case basic_json::value_t::array: - { - assert(m_it.array_iterator != m_object->m_value.array->end()); - return *m_it.array_iterator; - } - - case basic_json::value_t::null: - { - throw std::out_of_range("cannot get value"); - } - - default: - { - if (m_it.primitive_iterator.is_begin()) - { - return *m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief dereference the iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - pointer operator->() const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - assert(m_it.object_iterator != m_object->m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case basic_json::value_t::array: - { - assert(m_it.array_iterator != m_object->m_value.array->end()); - return &*m_it.array_iterator; - } - - default: - { - if (m_it.primitive_iterator.is_begin()) - { - return m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief post-increment (it++) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator++(int) - { - auto result = *this; - ++(*this); - return result; - } - - /*! - @brief pre-increment (++it) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator++() - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - default: - { - ++m_it.primitive_iterator; - break; - } - } - - return *this; - } - - /*! - @brief post-decrement (it--) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator--(int) - { - auto result = *this; - --(*this); - return result; - } - - /*! - @brief pre-decrement (--it) - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator--() - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - default: - { - --m_it.primitive_iterator; - break; - } - } - - return *this; - } - - /*! - @brief comparison: equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator==(const const_iterator& other) const - { - // if objects are not the same, the comparison is undefined - if (m_object != other.m_object) - { - throw std::domain_error("cannot compare iterators of different containers"); - } - - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - return (m_it.object_iterator == other.m_it.object_iterator); - } - - case basic_json::value_t::array: - { - return (m_it.array_iterator == other.m_it.array_iterator); - } - - default: - { - return (m_it.primitive_iterator == other.m_it.primitive_iterator); - } - } - } - - /*! - @brief comparison: not equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator!=(const const_iterator& other) const - { - return not operator==(other); - } - - /*! - @brief comparison: smaller - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator<(const const_iterator& other) const - { - // if objects are not the same, the comparison is undefined - if (m_object != other.m_object) - { - throw std::domain_error("cannot compare iterators of different containers"); - } - - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot compare order of object iterators"); - } - - case basic_json::value_t::array: - { - return (m_it.array_iterator < other.m_it.array_iterator); - } - - default: - { - return (m_it.primitive_iterator < other.m_it.primitive_iterator); - } - } - } - - /*! - @brief comparison: less than or equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator<=(const const_iterator& other) const - { - return not other.operator < (*this); - } - - /*! - @brief comparison: greater than - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator>(const const_iterator& other) const - { - return not operator<=(other); - } - - /*! - @brief comparison: greater than or equal - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - bool operator>=(const const_iterator& other) const - { - return not operator<(other); - } - - /*! - @brief add to iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator+=(difference_type i) - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use offsets with object iterators"); - } - - case basic_json::value_t::array: - { - std::advance(m_it.array_iterator, i); - break; - } - - default: - { - m_it.primitive_iterator += i; - break; - } - } - - return *this; - } - - /*! - @brief subtract from iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator& operator-=(difference_type i) - { - return operator+=(-i); - } - - /*! - @brief add to iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator+(difference_type i) - { - auto result = *this; - result += i; - return result; - } - - /*! - @brief subtract from iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - const_iterator operator-(difference_type i) - { - auto result = *this; - result -= i; - return result; - } - - /*! - @brief return difference - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - difference_type operator-(const const_iterator& other) const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use offsets with object iterators"); - } - - case basic_json::value_t::array: - { - return m_it.array_iterator - other.m_it.array_iterator; - } - - default: - { - return m_it.primitive_iterator - other.m_it.primitive_iterator; - } - } - } - - /*! - @brief access to successor - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference operator[](difference_type n) const - { - assert(m_object != nullptr); - - switch (m_object->m_type) - { - case basic_json::value_t::object: - { - throw std::domain_error("cannot use operator[] for object iterators"); - } - - case basic_json::value_t::array: - { - return *std::next(m_it.array_iterator, n); - } - - case basic_json::value_t::null: - { - throw std::out_of_range("cannot get value"); - } - - default: - { - if (m_it.primitive_iterator == -n) - { - return *m_object; - } - else - { - throw std::out_of_range("cannot get value"); - } - } - } - } - - /*! - @brief return the key of an object iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - typename object_t::key_type key() const - { - assert(m_object != nullptr); - - if (m_object->is_object()) - { - return m_it.object_iterator->first; - } - else - { - throw std::domain_error("cannot use key() for non-object iterators"); - } - } - - /*! - @brief return the value of an iterator - @pre The iterator is initialized; i.e. `m_object != nullptr`. - */ - reference value() const - { - return operator*(); - } - - private: - /// associated JSON instance - pointer m_object = nullptr; - /// the actual iterator of the associated instance - internal_iterator m_it = internal_iterator(); - }; - - /*! - @brief a mutable random access iterator for the @ref basic_json class - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): - It is possible to write to the pointed-to element. - - @since version 1.0.0 - */ - class iterator : public const_iterator - { - public: - using base_iterator = const_iterator; - using pointer = typename basic_json::pointer; - using reference = typename basic_json::reference; - - /// default constructor - iterator() = default; - - /// constructor for a given JSON instance - explicit iterator(pointer object) noexcept - : base_iterator(object) - {} - - /// copy constructor - iterator(const iterator& other) noexcept - : base_iterator(other) - {} - - /// copy assignment - iterator& operator=(iterator other) noexcept( - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value and - std::is_nothrow_move_constructible::value and - std::is_nothrow_move_assignable::value - ) - { - base_iterator::operator=(other); - return *this; - } - - /// return a reference to the value pointed to by the iterator - reference operator*() const - { - return const_cast(base_iterator::operator*()); - } - - /// dereference the iterator - pointer operator->() const - { - return const_cast(base_iterator::operator->()); - } - - /// post-increment (it++) - iterator operator++(int) - { - iterator result = *this; - base_iterator::operator++(); - return result; - } - - /// pre-increment (++it) - iterator& operator++() - { - base_iterator::operator++(); - return *this; - } - - /// post-decrement (it--) - iterator operator--(int) - { - iterator result = *this; - base_iterator::operator--(); - return result; - } - - /// pre-decrement (--it) - iterator& operator--() - { - base_iterator::operator--(); - return *this; - } - - /// add to iterator - iterator& operator+=(difference_type i) - { - base_iterator::operator+=(i); - return *this; - } - - /// subtract from iterator - iterator& operator-=(difference_type i) - { - base_iterator::operator-=(i); - return *this; - } - - /// add to iterator - iterator operator+(difference_type i) - { - auto result = *this; - result += i; - return result; - } - - /// subtract from iterator - iterator operator-(difference_type i) - { - auto result = *this; - result -= i; - return result; - } - - /// return difference - difference_type operator-(const iterator& other) const - { - return base_iterator::operator-(other); - } - - /// access to successor - reference operator[](difference_type n) const - { - return const_cast(base_iterator::operator[](n)); - } - - /// return the value of an iterator - reference value() const - { - return const_cast(base_iterator::value()); - } - }; - - /*! - @brief a template for a reverse iterator class - - @tparam Base the base iterator type to reverse. Valid types are @ref - iterator (to create @ref reverse_iterator) and @ref const_iterator (to - create @ref const_reverse_iterator). - - @requirement The class satisfies the following concept requirements: - - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): - The iterator that can be moved to point (forward and backward) to any - element in constant time. - - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): - It is possible to write to the pointed-to element (only if @a Base is - @ref iterator). - - @since version 1.0.0 - */ - template - class json_reverse_iterator : public std::reverse_iterator - { - public: - /// shortcut to the reverse iterator adaptor - using base_iterator = std::reverse_iterator; - /// the reference type for the pointed-to element - using reference = typename Base::reference; - - /// create reverse iterator from iterator - json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept - : base_iterator(it) - {} - - /// create reverse iterator from base class - json_reverse_iterator(const base_iterator& it) noexcept - : base_iterator(it) - {} - - /// post-increment (it++) - json_reverse_iterator operator++(int) - { - return base_iterator::operator++(1); - } - - /// pre-increment (++it) - json_reverse_iterator& operator++() - { - base_iterator::operator++(); - return *this; - } - - /// post-decrement (it--) - json_reverse_iterator operator--(int) - { - return base_iterator::operator--(1); - } - - /// pre-decrement (--it) - json_reverse_iterator& operator--() - { - base_iterator::operator--(); - return *this; - } - - /// add to iterator - json_reverse_iterator& operator+=(difference_type i) - { - base_iterator::operator+=(i); - return *this; - } - - /// add to iterator - json_reverse_iterator operator+(difference_type i) const - { - auto result = *this; - result += i; - return result; - } - - /// subtract from iterator - json_reverse_iterator operator-(difference_type i) const - { - auto result = *this; - result -= i; - return result; - } - - /// return difference - difference_type operator-(const json_reverse_iterator& other) const - { - return this->base() - other.base(); - } - - /// access to successor - reference operator[](difference_type n) const - { - return *(this->operator+(n)); - } - - /// return the key of an object iterator - typename object_t::key_type key() const - { - auto it = --this->base(); - return it.key(); - } - - /// return the value of an iterator - reference value() const - { - auto it = --this->base(); - return it.operator * (); - } - }; - - - private: - ////////////////////// - // lexer and parser // - ////////////////////// - - /*! - @brief lexical analysis - - This class organizes the lexical analysis during JSON deserialization. The - core of it is a scanner generated by [re2c](http://re2c.org) that - processes a buffer and recognizes tokens according to RFC 7159. - */ - class lexer - { - public: - /// token types for the parser - enum class token_type - { - uninitialized, ///< indicating the scanner is uninitialized - literal_true, ///< the `true` literal - literal_false, ///< the `false` literal - literal_null, ///< the `null` literal - value_string, ///< a string -- use get_string() for actual value - value_number, ///< a number -- use get_number() for actual value - begin_array, ///< the character for array begin `[` - begin_object, ///< the character for object begin `{` - end_array, ///< the character for array end `]` - end_object, ///< the character for object end `}` - name_separator, ///< the name separator `:` - value_separator, ///< the value separator `,` - parse_error, ///< indicating a parse error - end_of_input ///< indicating the end of the input buffer - }; - - /// the char type to use in the lexer - using lexer_char_t = unsigned char; - - /// constructor with a given buffer - explicit lexer(const string_t& s) noexcept - : m_stream(nullptr), m_buffer(s) - { - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + s.size(); - } - - /// constructor with a given stream - explicit lexer(std::istream* s) noexcept - : m_stream(s), m_buffer() - { - assert(m_stream != nullptr); - std::getline(*m_stream, m_buffer); - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_cursor = m_content; - m_limit = m_content + m_buffer.size(); - } - - /// default constructor - lexer() = default; - - // switch off unwanted functions - lexer(const lexer&) = delete; - lexer operator=(const lexer&) = delete; - - /*! - @brief create a string from one or two Unicode code points - - There are two cases: (1) @a codepoint1 is in the Basic Multilingual - Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) - @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to - represent a code point above U+FFFF. - - @param[in] codepoint1 the code point (can be high surrogate) - @param[in] codepoint2 the code point (can be low surrogate or 0) - - @return string representation of the code point; the length of the - result string is between 1 and 4 characters. - - @throw std::out_of_range if code point is > 0x10ffff; example: `"code - points above 0x10FFFF are invalid"` - @throw std::invalid_argument if the low surrogate is invalid; example: - `""missing or wrong low surrogate""` - - @complexity Constant. - - @see - */ - static string_t to_unicode(const std::size_t codepoint1, - const std::size_t codepoint2 = 0) - { - // calculate the code point from the given code points - std::size_t codepoint = codepoint1; - - // check if codepoint1 is a high surrogate - if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) - { - // check if codepoint2 is a low surrogate - if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) - { - codepoint = - // high surrogate occupies the most significant 22 bits - (codepoint1 << 10) - // low surrogate occupies the least significant 15 bits - + codepoint2 - // there is still the 0xD800, 0xDC00 and 0x10000 noise - // in the result so we have to subtract with: - // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 - - 0x35FDC00; - } - else - { - throw std::invalid_argument("missing or wrong low surrogate"); - } - } - - string_t result; - - if (codepoint < 0x80) - { - // 1-byte characters: 0xxxxxxx (ASCII) - result.append(1, static_cast(codepoint)); - } - else if (codepoint <= 0x7ff) - { - // 2-byte characters: 110xxxxx 10xxxxxx - result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0xffff) - { - // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); - result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0x10ffff) - { - // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); - result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - result.append(1, static_cast(0x80 | (codepoint & 0x3F))); - } - else - { - throw std::out_of_range("code points above 0x10FFFF are invalid"); - } - - return result; - } - - /// return name of values of type token_type (only used for errors) - static std::string token_type_name(const token_type t) - { - switch (t) - { - case token_type::uninitialized: - return ""; - case token_type::literal_true: - return "true literal"; - case token_type::literal_false: - return "false literal"; - case token_type::literal_null: - return "null literal"; - case token_type::value_string: - return "string literal"; - case token_type::value_number: - return "number literal"; - case token_type::begin_array: - return "'['"; - case token_type::begin_object: - return "'{'"; - case token_type::end_array: - return "']'"; - case token_type::end_object: - return "'}'"; - case token_type::name_separator: - return "':'"; - case token_type::value_separator: - return "','"; - case token_type::parse_error: - return ""; - case token_type::end_of_input: - return "end of input"; - default: - { - // catch non-enum values - return "unknown token"; // LCOV_EXCL_LINE - } - } - } - - /*! - This function implements a scanner for JSON. It is specified using - regular expressions that try to follow RFC 7159 as close as possible. - These regular expressions are then translated into a minimized - deterministic finite automaton (DFA) by the tool - [re2c](http://re2c.org). As a result, the translated code for this - function consists of a large block of code with `goto` jumps. - - @return the class of the next token read from the buffer - - @complexity Linear in the length of the input.\n - - Proposition: The loop below will always terminate for finite input.\n - - Proof (by contradiction): Assume a finite input. To loop forever, the - loop must never hit code with a `break` statement. The only code - snippets without a `break` statement are the continue statements for - whitespace and byte-order-marks. To loop forever, the input must be an - infinite sequence of whitespace or byte-order-marks. This contradicts - the assumption of finite input, q.e.d. - */ - token_type scan() noexcept - { - while (true) - { - // pointer for backtracking information - m_marker = nullptr; - - // remember the begin of the token - m_start = m_cursor; - assert(m_start != nullptr); - - - { - lexer_char_t yych; - unsigned int yyaccept = 0; - static const unsigned char yybm[] = - { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 32, 32, 0, 0, 32, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 160, 128, 0, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - }; - if ((m_limit - m_cursor) < 5) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 32) - { - goto basic_json_parser_6; - } - if (yych <= '\\') - { - if (yych <= '-') - { - if (yych <= '"') - { - if (yych <= 0x00) - { - goto basic_json_parser_2; - } - if (yych <= '!') - { - goto basic_json_parser_4; - } - goto basic_json_parser_9; - } - else - { - if (yych <= '+') - { - goto basic_json_parser_4; - } - if (yych <= ',') - { - goto basic_json_parser_10; - } - goto basic_json_parser_12; - } - } - else - { - if (yych <= '9') - { - if (yych <= '/') - { - goto basic_json_parser_4; - } - if (yych <= '0') - { - goto basic_json_parser_13; - } - goto basic_json_parser_15; - } - else - { - if (yych <= ':') - { - goto basic_json_parser_17; - } - if (yych == '[') - { - goto basic_json_parser_19; - } - goto basic_json_parser_4; - } - } - } - else - { - if (yych <= 't') - { - if (yych <= 'f') - { - if (yych <= ']') - { - goto basic_json_parser_21; - } - if (yych <= 'e') - { - goto basic_json_parser_4; - } - goto basic_json_parser_23; - } - else - { - if (yych == 'n') - { - goto basic_json_parser_24; - } - if (yych <= 's') - { - goto basic_json_parser_4; - } - goto basic_json_parser_25; - } - } - else - { - if (yych <= '|') - { - if (yych == '{') - { - goto basic_json_parser_26; - } - goto basic_json_parser_4; - } - else - { - if (yych <= '}') - { - goto basic_json_parser_28; - } - if (yych == 0xEF) - { - goto basic_json_parser_30; - } - goto basic_json_parser_4; - } - } - } -basic_json_parser_2: - ++m_cursor; - { - last_token_type = token_type::end_of_input; - break; - } -basic_json_parser_4: - ++m_cursor; -basic_json_parser_5: - { - last_token_type = token_type::parse_error; - break; - } -basic_json_parser_6: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 32) - { - goto basic_json_parser_6; - } - { - continue; - } -basic_json_parser_9: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych <= 0x1F) - { - goto basic_json_parser_5; - } - goto basic_json_parser_32; -basic_json_parser_10: - ++m_cursor; - { - last_token_type = token_type::value_separator; - break; - } -basic_json_parser_12: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_5; - } - if (yych <= '0') - { - goto basic_json_parser_13; - } - if (yych <= '9') - { - goto basic_json_parser_15; - } - goto basic_json_parser_5; -basic_json_parser_13: - yyaccept = 1; - yych = *(m_marker = ++m_cursor); - if (yych <= 'D') - { - if (yych == '.') - { - goto basic_json_parser_37; - } - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - } -basic_json_parser_14: - { - last_token_type = token_type::value_number; - break; - } -basic_json_parser_15: - yyaccept = 1; - m_marker = ++m_cursor; - if ((m_limit - m_cursor) < 3) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yybm[0 + yych] & 64) - { - goto basic_json_parser_15; - } - if (yych <= 'D') - { - if (yych == '.') - { - goto basic_json_parser_37; - } - goto basic_json_parser_14; - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - goto basic_json_parser_14; - } -basic_json_parser_17: - ++m_cursor; - { - last_token_type = token_type::name_separator; - break; - } -basic_json_parser_19: - ++m_cursor; - { - last_token_type = token_type::begin_array; - break; - } -basic_json_parser_21: - ++m_cursor; - { - last_token_type = token_type::end_array; - break; - } -basic_json_parser_23: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'a') - { - goto basic_json_parser_39; - } - goto basic_json_parser_5; -basic_json_parser_24: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'u') - { - goto basic_json_parser_40; - } - goto basic_json_parser_5; -basic_json_parser_25: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 'r') - { - goto basic_json_parser_41; - } - goto basic_json_parser_5; -basic_json_parser_26: - ++m_cursor; - { - last_token_type = token_type::begin_object; - break; - } -basic_json_parser_28: - ++m_cursor; - { - last_token_type = token_type::end_object; - break; - } -basic_json_parser_30: - yyaccept = 0; - yych = *(m_marker = ++m_cursor); - if (yych == 0xBB) - { - goto basic_json_parser_42; - } - goto basic_json_parser_5; -basic_json_parser_31: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; -basic_json_parser_32: - if (yybm[0 + yych] & 128) - { - goto basic_json_parser_31; - } - if (yych <= 0x1F) - { - goto basic_json_parser_33; - } - if (yych <= '"') - { - goto basic_json_parser_34; - } - goto basic_json_parser_36; -basic_json_parser_33: - m_cursor = m_marker; - if (yyaccept == 0) - { - goto basic_json_parser_5; - } - else - { - goto basic_json_parser_14; - } -basic_json_parser_34: - ++m_cursor; - { - last_token_type = token_type::value_string; - break; - } -basic_json_parser_36: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= 'e') - { - if (yych <= '/') - { - if (yych == '"') - { - goto basic_json_parser_31; - } - if (yych <= '.') - { - goto basic_json_parser_33; - } - goto basic_json_parser_31; - } - else - { - if (yych <= '\\') - { - if (yych <= '[') - { - goto basic_json_parser_33; - } - goto basic_json_parser_31; - } - else - { - if (yych == 'b') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - } - } - else - { - if (yych <= 'q') - { - if (yych <= 'f') - { - goto basic_json_parser_31; - } - if (yych == 'n') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 's') - { - if (yych <= 'r') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 't') - { - goto basic_json_parser_31; - } - if (yych <= 'u') - { - goto basic_json_parser_43; - } - goto basic_json_parser_33; - } - } - } -basic_json_parser_37: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_44; - } - goto basic_json_parser_33; -basic_json_parser_38: - yych = *++m_cursor; - if (yych <= ',') - { - if (yych == '+') - { - goto basic_json_parser_46; - } - goto basic_json_parser_33; - } - else - { - if (yych <= '-') - { - goto basic_json_parser_46; - } - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_47; - } - goto basic_json_parser_33; - } -basic_json_parser_39: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_49; - } - goto basic_json_parser_33; -basic_json_parser_40: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_50; - } - goto basic_json_parser_33; -basic_json_parser_41: - yych = *++m_cursor; - if (yych == 'u') - { - goto basic_json_parser_51; - } - goto basic_json_parser_33; -basic_json_parser_42: - yych = *++m_cursor; - if (yych == 0xBF) - { - goto basic_json_parser_52; - } - goto basic_json_parser_33; -basic_json_parser_43: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_54; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_54; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_54; - } - goto basic_json_parser_33; - } -basic_json_parser_44: - yyaccept = 1; - m_marker = ++m_cursor; - if ((m_limit - m_cursor) < 3) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= 'D') - { - if (yych <= '/') - { - goto basic_json_parser_14; - } - if (yych <= '9') - { - goto basic_json_parser_44; - } - goto basic_json_parser_14; - } - else - { - if (yych <= 'E') - { - goto basic_json_parser_38; - } - if (yych == 'e') - { - goto basic_json_parser_38; - } - goto basic_json_parser_14; - } -basic_json_parser_46: - yych = *++m_cursor; - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych >= ':') - { - goto basic_json_parser_33; - } -basic_json_parser_47: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '/') - { - goto basic_json_parser_14; - } - if (yych <= '9') - { - goto basic_json_parser_47; - } - goto basic_json_parser_14; -basic_json_parser_49: - yych = *++m_cursor; - if (yych == 's') - { - goto basic_json_parser_55; - } - goto basic_json_parser_33; -basic_json_parser_50: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_56; - } - goto basic_json_parser_33; -basic_json_parser_51: - yych = *++m_cursor; - if (yych == 'e') - { - goto basic_json_parser_58; - } - goto basic_json_parser_33; -basic_json_parser_52: - ++m_cursor; - { - continue; - } -basic_json_parser_54: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_60; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_60; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_60; - } - goto basic_json_parser_33; - } -basic_json_parser_55: - yych = *++m_cursor; - if (yych == 'e') - { - goto basic_json_parser_61; - } - goto basic_json_parser_33; -basic_json_parser_56: - ++m_cursor; - { - last_token_type = token_type::literal_null; - break; - } -basic_json_parser_58: - ++m_cursor; - { - last_token_type = token_type::literal_true; - break; - } -basic_json_parser_60: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_63; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_63; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_63; - } - goto basic_json_parser_33; - } -basic_json_parser_61: - ++m_cursor; - { - last_token_type = token_type::literal_false; - break; - } -basic_json_parser_63: - ++m_cursor; - if (m_limit <= m_cursor) - { - yyfill(); // LCOV_EXCL_LINE; - } - yych = *m_cursor; - if (yych <= '@') - { - if (yych <= '/') - { - goto basic_json_parser_33; - } - if (yych <= '9') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - else - { - if (yych <= 'F') - { - goto basic_json_parser_31; - } - if (yych <= '`') - { - goto basic_json_parser_33; - } - if (yych <= 'f') - { - goto basic_json_parser_31; - } - goto basic_json_parser_33; - } - } - - } - - return last_token_type; - } - - /// append data from the stream to the internal buffer - void yyfill() noexcept - { - if (m_stream == nullptr or not * m_stream) - { - return; - } - - const auto offset_start = m_start - m_content; - const auto offset_marker = m_marker - m_start; - const auto offset_cursor = m_cursor - m_start; - - m_buffer.erase(0, static_cast(offset_start)); - std::string line; - assert(m_stream != nullptr); - std::getline(*m_stream, line); - m_buffer += "\n" + line; // add line with newline symbol - - m_content = reinterpret_cast(m_buffer.c_str()); - assert(m_content != nullptr); - m_start = m_content; - m_marker = m_start + offset_marker; - m_cursor = m_start + offset_cursor; - m_limit = m_start + m_buffer.size() - 1; - } - - /// return string representation of last read token - string_t get_token_string() const - { - assert(m_start != nullptr); - return string_t(reinterpret_cast(m_start), - static_cast(m_cursor - m_start)); - } - - /*! - @brief return string value for string tokens - - The function iterates the characters between the opening and closing - quotes of the string value. The complete string is the range - [m_start,m_cursor). Consequently, we iterate from m_start+1 to - m_cursor-1. - - We differentiate two cases: - - 1. Escaped characters. In this case, a new character is constructed - according to the nature of the escape. Some escapes create new - characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied - as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape - `"\\uxxxx"` need special care. In this case, to_unicode takes care - of the construction of the values. - 2. Unescaped characters are copied as is. - - @pre `m_cursor - m_start >= 2`, meaning the length of the last token - is at least 2 bytes which is trivially true for any string (which - consists of at least two quotes). - - " c1 c2 c3 ... " - ^ ^ - m_start m_cursor - - @complexity Linear in the length of the string.\n - - Lemma: The loop body will always terminate.\n - - Proof (by contradiction): Assume the loop body does not terminate. As - the loop body does not contain another loop, one of the called - functions must never return. The called functions are `std::strtoul` - and to_unicode. Neither function can loop forever, so the loop body - will never loop forever which contradicts the assumption that the loop - body does not terminate, q.e.d.\n - - Lemma: The loop condition for the for loop is eventually false.\n - - Proof (by contradiction): Assume the loop does not terminate. Due to - the above lemma, this can only be due to a tautological loop - condition; that is, the loop condition i < m_cursor - 1 must always be - true. Let x be the change of i for any loop iteration. Then - m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This - can be rephrased to m_cursor - m_start - 2 > x. With the - precondition, we x <= 0, meaning that the loop condition holds - indefinitly if i is always decreased. However, observe that the value - of i is strictly increasing with each iteration, as it is incremented - by 1 in the iteration expression and never decremented inside the loop - body. Hence, the loop condition will eventually be false which - contradicts the assumption that the loop condition is a tautology, - q.e.d. - - @return string value of current token without opening and closing - quotes - @throw std::out_of_range if to_unicode fails - */ - string_t get_string() const - { - assert(m_cursor - m_start >= 2); - - string_t result; - result.reserve(static_cast(m_cursor - m_start - 2)); - - // iterate the result between the quotes - for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) - { - // process escaped characters - if (*i == '\\') - { - // read next character - ++i; - - switch (*i) - { - // the default escapes - case 't': - { - result += "\t"; - break; - } - case 'b': - { - result += "\b"; - break; - } - case 'f': - { - result += "\f"; - break; - } - case 'n': - { - result += "\n"; - break; - } - case 'r': - { - result += "\r"; - break; - } - case '\\': - { - result += "\\"; - break; - } - case '/': - { - result += "/"; - break; - } - case '"': - { - result += "\""; - break; - } - - // unicode - case 'u': - { - // get code xxxx from uxxxx - auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), - 4).c_str(), nullptr, 16); - - // check if codepoint is a high surrogate - if (codepoint >= 0xD800 and codepoint <= 0xDBFF) - { - // make sure there is a subsequent unicode - if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') - { - throw std::invalid_argument("missing low surrogate"); - } - - // get code yyyy from uxxxx\uyyyy - auto codepoint2 = std::strtoul(std::string(reinterpret_cast - (i + 7), 4).c_str(), nullptr, 16); - result += to_unicode(codepoint, codepoint2); - // skip the next 10 characters (xxxx\uyyyy) - i += 10; - } - else - { - // add unicode character(s) - result += to_unicode(codepoint); - // skip the next four characters (xxxx) - i += 4; - } - break; - } - } - } - else - { - // all other characters are just copied to the end of the - // string - result.append(1, static_cast(*i)); - } - } - - return result; - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - long double str_to_float_t(long double* /* type */, char** endptr) const - { - return std::strtold(reinterpret_cast(m_start), endptr); - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } - - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in] type the @ref number_float_t in use - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } - - /*! - @brief return number value for number tokens - - This function translates the last token into the most appropriate - number type (either integer, unsigned integer or floating point), - which is passed back to the caller via the result parameter. - - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. - - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). - - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. - */ - void get_number(basic_json& result) const - { - assert(m_start != nullptr); - - const lexer::lexer_char_t* curptr = m_start; - - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; - - // maximum absolute value of the relevant integer type - number_unsigned_t max; - - // temporarily store the type to avoid unecessary bitfield access - value_t type; - - // look for sign - if (*curptr == '-') - { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; - } - else - { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); - } - - // count the significant figures - for (; curptr < m_cursor; curptr++) - { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') - { - if (*curptr == '.') - { - // don't count '.' but change to float - type = value_t::number_float; - continue; - } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; - break; - } - - // skip if definitely not an integer - if (type != value_t::number_float) - { - // multiply last value by ten and add the new digit - auto temp = value * 10 + *curptr - '0'; - - // test for overflow - if (temp < value || temp > max) - { - // overflow - type = value_t::number_float; - } - else - { - // no overflow - save it - value = temp; - } - } - } - - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - result.m_value.number_integer = -static_cast(value); - } - else - { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); - } - - // save the type - result.m_type = type; - } - - private: - /// optional input stream - std::istream* m_stream = nullptr; - /// the buffer - string_t m_buffer; - /// the buffer pointer - const lexer_char_t* m_content = nullptr; - /// pointer to the beginning of the current symbol - const lexer_char_t* m_start = nullptr; - /// pointer for backtracking information - const lexer_char_t* m_marker = nullptr; - /// pointer to the current symbol - const lexer_char_t* m_cursor = nullptr; - /// pointer to the end of the buffer - const lexer_char_t* m_limit = nullptr; - /// the last token type - token_type last_token_type = token_type::end_of_input; - }; - - /*! - @brief syntax analysis - - This class implements a recursive decent parser. - */ - class parser - { - public: - /// constructor for strings - parser(const string_t& s, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(s) - { - // read first token - get_token(); - } - - /// a parser reading from an input stream - parser(std::istream& _is, const parser_callback_t cb = nullptr) noexcept - : callback(cb), m_lexer(&_is) - { - // read first token - get_token(); - } - - /// public parser interface - basic_json parse() - { - basic_json result = parse_internal(true); - result.assert_invariant(); - - expect(lexer::token_type::end_of_input); - - // return parser result and replace it with null in case the - // top-level value was discarded by the callback function - return result.is_discarded() ? basic_json() : std::move(result); - } - - private: - /// the actual parser - basic_json parse_internal(bool keep) - { - auto result = basic_json(value_t::discarded); - - switch (last_token) - { - case lexer::token_type::begin_object: - { - if (keep and (not callback or (keep = callback(depth++, parse_event_t::object_start, result)))) - { - // explicitly set result to object to cope with {} - result.m_type = value_t::object; - result.m_value = value_t::object; - } - - // read next token - get_token(); - - // closing } -> we are done - if (last_token == lexer::token_type::end_object) - { - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - // no comma is expected here - unexpect(lexer::token_type::value_separator); - - // otherwise: parse key-value pairs - do - { - // ugly, but could be fixed with loop reorganization - if (last_token == lexer::token_type::value_separator) - { - get_token(); - } - - // store key - expect(lexer::token_type::value_string); - const auto key = m_lexer.get_string(); - - bool keep_tag = false; - if (keep) - { - if (callback) - { - basic_json k(key); - keep_tag = callback(depth, parse_event_t::key, k); - } - else - { - keep_tag = true; - } - } - - // parse separator (:) - get_token(); - expect(lexer::token_type::name_separator); - - // parse and add value - get_token(); - auto value = parse_internal(keep); - if (keep and keep_tag and not value.is_discarded()) - { - result[key] = std::move(value); - } - } - while (last_token == lexer::token_type::value_separator); - - // closing } - expect(lexer::token_type::end_object); - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result = basic_json(value_t::discarded); - } - - return result; - } - - case lexer::token_type::begin_array: - { - if (keep and (not callback or (keep = callback(depth++, parse_event_t::array_start, result)))) - { - // explicitly set result to object to cope with [] - result.m_type = value_t::array; - result.m_value = value_t::array; - } - - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == lexer::token_type::end_array) - { - get_token(); - if (callback and not callback(--depth, parse_event_t::array_end, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - // no comma is expected here - unexpect(lexer::token_type::value_separator); - - // otherwise: parse values - do - { - // ugly, but could be fixed with loop reorganization - if (last_token == lexer::token_type::value_separator) - { - get_token(); - } - - // parse value - auto value = parse_internal(keep); - if (keep and not value.is_discarded()) - { - result.push_back(std::move(value)); - } - } - while (last_token == lexer::token_type::value_separator); - - // closing ] - expect(lexer::token_type::end_array); - get_token(); - if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) - { - result = basic_json(value_t::discarded); - } - - return result; - } - - case lexer::token_type::literal_null: - { - get_token(); - result.m_type = value_t::null; - break; - } - - case lexer::token_type::value_string: - { - const auto s = m_lexer.get_string(); - get_token(); - result = basic_json(s); - break; - } - - case lexer::token_type::literal_true: - { - get_token(); - result.m_type = value_t::boolean; - result.m_value = true; - break; - } - - case lexer::token_type::literal_false: - { - get_token(); - result.m_type = value_t::boolean; - result.m_value = false; - break; - } - - case lexer::token_type::value_number: - { - m_lexer.get_number(result); - get_token(); - break; - } - - default: - { - // the last token was unexpected - unexpect(last_token); - } - } - - if (keep and callback and not callback(depth, parse_event_t::value, result)) - { - result = basic_json(value_t::discarded); - } - return result; - } - - /// get next token from lexer - typename lexer::token_type get_token() noexcept - { - last_token = m_lexer.scan(); - return last_token; - } - - void expect(typename lexer::token_type t) const - { - if (t != last_token) - { - std::string error_msg = "parse error - unexpected "; - error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + - "'") : - lexer::token_type_name(last_token)); - error_msg += "; expected " + lexer::token_type_name(t); - throw std::invalid_argument(error_msg); - } - } - - void unexpect(typename lexer::token_type t) const - { - if (t == last_token) - { - std::string error_msg = "parse error - unexpected "; - error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + - "'") : - lexer::token_type_name(last_token)); - throw std::invalid_argument(error_msg); - } - } - - private: - /// current level of recursion - int depth = 0; - /// callback function - const parser_callback_t callback = nullptr; - /// the type of the last read token - typename lexer::token_type last_token = lexer::token_type::uninitialized; - /// the lexer - lexer m_lexer; - }; - - public: - /*! - @brief JSON Pointer - - A JSON pointer defines a string syntax for identifying a specific value - within a JSON document. It can be used with functions `at` and - `operator[]`. Furthermore, JSON pointers are the base for JSON patches. - - @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - - @since version 2.0.0 - */ - class json_pointer - { - /// allow basic_json to access private members - friend class basic_json; - - public: - /*! - @brief create JSON pointer - - Create a JSON pointer according to the syntax described in - [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). - - @param[in] s string representing the JSON pointer; if omitted, the - empty string is assumed which references the whole JSON - value - - @throw std::domain_error if reference token is nonempty and does not - begin with a slash (`/`); example: `"JSON pointer must be empty or - begin with /"` - @throw std::domain_error if a tilde (`~`) is not followed by `0` - (representing `~`) or `1` (representing `/`); example: `"escape error: - ~ must be followed with 0 or 1"` - - @liveexample{The example shows the construction several valid JSON - pointers as well as the exceptional behavior.,json_pointer} - - @since version 2.0.0 - */ - explicit json_pointer(const std::string& s = "") - : reference_tokens(split(s)) - {} - - /*! - @brief return a string representation of the JSON pointer - - @invariant For each JSON pointer `ptr`, it holds: - @code {.cpp} - ptr == json_pointer(ptr.to_string()); - @endcode - - @return a string representation of the JSON pointer - - @liveexample{The example shows the result of `to_string`., - json_pointer__to_string} - - @since version 2.0.0 - */ - std::string to_string() const noexcept - { - return std::accumulate(reference_tokens.begin(), - reference_tokens.end(), std::string{}, - [](const std::string & a, const std::string & b) - { - return a + "/" + escape(b); - }); - } - - /// @copydoc to_string() - operator std::string() const - { - return to_string(); - } - - private: - /// remove and return last reference pointer - std::string pop_back() - { - if (is_root()) - { - throw std::domain_error("JSON pointer has no parent"); - } - - auto last = reference_tokens.back(); - reference_tokens.pop_back(); - return last; - } - - /// return whether pointer points to the root document - bool is_root() const - { - return reference_tokens.empty(); - } - - json_pointer top() const - { - if (is_root()) - { - throw std::domain_error("JSON pointer has no parent"); - } - - json_pointer result = *this; - result.reference_tokens = {reference_tokens[0]}; - return result; - } - - /*! - @brief create and return a reference to the pointed to value - - @complexity Linear in the number of reference tokens. - */ - reference get_and_create(reference j) const - { - pointer result = &j; - - // in case no reference tokens exist, return a reference to the - // JSON value j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case value_t::array: - { - // create an entry in the array - result = &result->operator[](static_cast(std::stoi(reference_token))); - break; - } - - /* - The following code is only reached if there exists a - reference token _and_ the current value is primitive. In - this case, we have an error situation, because primitive - values may only occur as single value; that is, with an - empty list of reference tokens. - */ - default: - { - throw std::domain_error("invalid value to unflatten"); - } - } - } - - return *result; - } - - /*! - @brief return a reference to the pointed to value - - @param[in] ptr a JSON value - - @return reference to the JSON value pointed to by the JSON pointer - - @complexity Linear in the length of the JSON pointer. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - */ - reference get_unchecked(pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case value_t::array: - { - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - if (reference_token == "-") - { - // explicityly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); - } - else - { - // convert array index to number; unchecked access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); - } - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - reference get_checked(pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - /*! - @brief return a const reference to the pointed to value - - @param[in] ptr a JSON value - - @return const reference to the JSON value pointed to by the JSON - pointer - */ - const_reference get_unchecked(const_pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" cannot be used for const access - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // use unchecked array access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - const_reference get_checked(const_pointer ptr) const - { - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case value_t::array: - { - if (reference_token == "-") - { - // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (reference_token.size() > 1 and reference_token[0] == '0') - { - throw std::domain_error("array index must not begin with '0'"); - } - - // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); - break; - } - - default: - { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); - } - } - } - - return *ptr; - } - - /// split the string input to reference tokens - static std::vector split(std::string reference_string) - { - std::vector result; - - // special case: empty reference string -> no reference tokens - if (reference_string.empty()) - { - return result; - } - - // check if nonempty reference string begins with slash - if (reference_string[0] != '/') - { - throw std::domain_error("JSON pointer must be empty or begin with '/'"); - } - - // extract the reference tokens: - // - slash: position of the last read slash (or end of string) - // - start: position after the previous slash - for ( - // search for the first slash after the first character - size_t slash = reference_string.find_first_of("/", 1), - // set the beginning of the first reference token - start = 1; - // we can stop if start == string::npos+1 = 0 - start != 0; - // set the beginning of the next reference token - // (will eventually be 0 if slash == std::string::npos) - start = slash + 1, - // find next slash - slash = reference_string.find_first_of("/", start)) - { - // use the text between the beginning of the reference token - // (start) and the last slash (slash). - auto reference_token = reference_string.substr(start, slash - start); - - // check reference tokens are properly escaped - for (size_t pos = reference_token.find_first_of("~"); - pos != std::string::npos; - pos = reference_token.find_first_of("~", pos + 1)) - { - assert(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (pos == reference_token.size() - 1 or - (reference_token[pos + 1] != '0' and - reference_token[pos + 1] != '1')) - { - throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); - } - } - - // finally, store the reference token - unescape(reference_token); - result.push_back(reference_token); - } - - return result; - } - - private: - /*! - @brief replace all occurrences of a substring by another string - - @param[in,out] s the string to manipulate - @param[in] f the substring to replace with @a t - @param[in] t the string to replace @a f - - @return The string @a s where all occurrences of @a f are replaced - with @a t. - - @pre The search string @a f must not be empty. - - @since version 2.0.0 - */ - static void replace_substring(std::string& s, - const std::string& f, - const std::string& t) - { - assert(not f.empty()); - - for ( - size_t pos = s.find(f); // find first occurrence of f - pos != std::string::npos; // make sure f was found - s.replace(pos, f.size(), t), // replace with t - pos = s.find(f, pos + t.size()) // find next occurrence of f - ); - } - - /// escape tilde and slash - static std::string escape(std::string s) - { - // escape "~"" to "~0" and "/" to "~1" - replace_substring(s, "~", "~0"); - replace_substring(s, "/", "~1"); - return s; - } - - /// unescape tilde and slash - static void unescape(std::string& s) - { - // first transform any occurrence of the sequence '~1' to '/' - replace_substring(s, "~1", "/"); - // then transform any occurrence of the sequence '~0' to '~' - replace_substring(s, "~0", "~"); - } - - /*! - @param[in] reference_string the reference string to the current value - @param[in] value the value to consider - @param[in,out] result the result object to insert values to - - @note Empty objects or arrays are flattened to `null`. - */ - static void flatten(const std::string& reference_string, - const basic_json& value, - basic_json& result) - { - switch (value.m_type) - { - case value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), - element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } - } - - /*! - @param[in] value flattened JSON - - @return unflattened JSON - */ - static basic_json unflatten(const basic_json& value) - { - if (not value.is_object()) - { - throw std::domain_error("only objects can be unflattened"); - } - - basic_json result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (not element.second.is_primitive()) - { - throw std::domain_error("values in object must be primitive"); - } - - // assign value to reference pointed to by JSON pointer; Note - // that if the JSON pointer is "" (i.e., points to the whole - // value), function get_and_create returns a reference to - // result itself. An assignment will then create a primitive - // value. - json_pointer(element.first).get_and_create(result) = element.second; - } - - return result; - } - - private: - /// the reference tokens - std::vector reference_tokens {}; - }; - - ////////////////////////// - // JSON Pointer support // - ////////////////////////// - - /// @name JSON Pointer functions - /// @{ - - /*! - @brief access specified element via JSON Pointer - - Uses a JSON pointer to retrieve a reference to the respective JSON value. - No bound checking is performed. Similar to @ref operator[](const typename - object_t::key_type&), `null` values are created in arrays and objects if - necessary. - - In particular: - - If the JSON pointer points to an object key that does not exist, it - is created an filled with a `null` value before a reference to it - is returned. - - If the JSON pointer points to an array index that does not exist, it - is created an filled with a `null` value before a reference to it - is returned. All indices between the current maximum and the given - index are also filled with `null`. - - The special value `-` is treated as a synonym for the index past the - end. - - @param[in] ptr a JSON pointer - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,operatorjson_pointer} - - @since version 2.0.0 - */ - reference operator[](const json_pointer& ptr) - { - return ptr.get_unchecked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Uses a JSON pointer to retrieve a reference to the respective JSON value. - No bound checking is performed. The function does not change the JSON - value; no `null` values are created. In particular, the the special value - `-` yields an exception. - - @param[in] ptr JSON pointer to the desired element - - @return const reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} - - @since version 2.0.0 - */ - const_reference operator[](const json_pointer& ptr) const - { - return ptr.get_unchecked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Returns a reference to the element at with specified JSON pointer @a ptr, - with bounds checking. - - @param[in] ptr JSON pointer to the desired element - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,at_json_pointer} - - @since version 2.0.0 - */ - reference at(const json_pointer& ptr) - { - return ptr.get_checked(this); - } - - /*! - @brief access specified element via JSON Pointer - - Returns a const reference to the element at with specified JSON pointer @a - ptr, with bounds checking. - - @param[in] ptr JSON pointer to the desired element - - @return reference to the element pointed to by @a ptr - - @complexity Constant. - - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number - - @liveexample{The behavior is shown in the example.,at_json_pointer_const} - - @since version 2.0.0 - */ - const_reference at(const json_pointer& ptr) const - { - return ptr.get_checked(this); - } - - /*! - @brief return flattened JSON value - - The function creates a JSON object whose keys are JSON pointers (see [RFC - 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all - primitive. The original JSON value can be restored using the @ref - unflatten() function. - - @return an object that maps JSON pointers to primitve values - - @note Empty objects and arrays are flattened to `null` and will not be - reconstructed correctly by the @ref unflatten() function. - - @complexity Linear in the size the JSON value. - - @liveexample{The following code shows how a JSON object is flattened to an - object whose keys consist of JSON pointers.,flatten} - - @sa @ref unflatten() for the reverse function - - @since version 2.0.0 - */ - basic_json flatten() const - { - basic_json result(value_t::object); - json_pointer::flatten("", *this, result); - return result; - } - - /*! - @brief unflatten a previously flattened JSON value - - The function restores the arbitrary nesting of a JSON value that has been - flattened before using the @ref flatten() function. The JSON value must - meet certain constraints: - 1. The value must be an object. - 2. The keys must be JSON pointers (see - [RFC 6901](https://tools.ietf.org/html/rfc6901)) - 3. The mapped values must be primitive JSON types. - - @return the original JSON from a flattened version - - @note Empty objects and arrays are flattened by @ref flatten() to `null` - values and can not unflattened to their original type. Apart from - this example, for a JSON value `j`, the following is always true: - `j == j.flatten().unflatten()`. - - @complexity Linear in the size the JSON value. - - @liveexample{The following code shows how a flattened JSON object is - unflattened into the original nested JSON object.,unflatten} - - @sa @ref flatten() for the reverse function - - @since version 2.0.0 - */ - basic_json unflatten() const - { - return json_pointer::unflatten(*this); - } - - /// @} - - ////////////////////////// - // JSON Patch functions // - ////////////////////////// - - /// @name JSON Patch functions - /// @{ - - /*! - @brief applies a JSON patch - - [JSON Patch](http://jsonpatch.com) defines a JSON document structure for - expressing a sequence of operations to apply to a JSON) document. With - this funcion, a JSON Patch is applied to the current JSON value by - executing all operations from the patch. - - @param[in] json_patch JSON patch document - @return patched document - - @note The application of a patch is atomic: Either all operations succeed - and the patched document is returned or an exception is thrown. In - any case, the original value is not changed: the patch is applied - to a copy of the value. - - @throw std::out_of_range if a JSON pointer inside the patch could not - be resolved successfully in the current JSON value; example: `"key baz - not found"` - @throw invalid_argument if the JSON patch is malformed (e.g., mandatory - attributes are missing); example: `"operation add must have member path"` - - @complexity Linear in the size of the JSON value and the length of the - JSON patch. As usually only a fraction of the JSON value is affected by - the patch, the complexity can usually be neglected. - - @liveexample{The following code shows how a JSON patch is applied to a - value.,patch} - - @sa @ref diff -- create a JSON patch by comparing two JSON values - - @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) - @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) - - @since version 2.0.0 - */ - basic_json patch(const basic_json& json_patch) const - { - // make a working copy to apply the patch to - basic_json result = *this; - - // the valid JSON Patch operations - enum class patch_operations {add, remove, replace, move, copy, test, invalid}; - - const auto get_op = [](const std::string op) - { - if (op == "add") - { - return patch_operations::add; - } - if (op == "remove") - { - return patch_operations::remove; - } - if (op == "replace") - { - return patch_operations::replace; - } - if (op == "move") - { - return patch_operations::move; - } - if (op == "copy") - { - return patch_operations::copy; - } - if (op == "test") - { - return patch_operations::test; - } - - return patch_operations::invalid; - }; - - // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, basic_json val) - { - // adding to the root of the target document means replacing it - if (ptr.is_root()) - { - result = val; - } - else - { - // make sure the top element of the pointer exists - json_pointer top_pointer = ptr.top(); - if (top_pointer != ptr) - { - basic_json& x = result.at(top_pointer); - } - - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result[ptr]; - - switch (parent.m_type) - { - case value_t::null: - case value_t::object: - { - // use operator[] to add value - parent[last_path] = val; - break; - } - - case value_t::array: - { - if (last_path == "-") - { - // special case: append to back - parent.push_back(val); - } - else - { - const auto idx = std::stoi(last_path); - if (static_cast(idx) > parent.size()) - { - // avoid undefined behavior - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); - } - else - { - // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); - } - } - break; - } - - default: - { - // if there exists a parent it cannot be primitive - assert(false); // LCOV_EXCL_LINE - } - } - } - }; - - // wrapper for "remove" operation; remove value at ptr - const auto operation_remove = [&result](json_pointer & ptr) - { - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result.at(ptr); - - // remove child - if (parent.is_object()) - { - // perform range check - auto it = parent.find(last_path); - if (it != parent.end()) - { - parent.erase(it); - } - else - { - throw std::out_of_range("key '" + last_path + "' not found"); - } - } - else if (parent.is_array()) - { - // note erase performs range check - parent.erase(static_cast(std::stoi(last_path))); - } - }; - - // type check - if (not json_patch.is_array()) - { - // a JSON patch must be an array of objects - throw std::invalid_argument("JSON patch must be an array of objects"); - } - - // iterate and apply th eoperations - for (const auto& val : json_patch) - { - // wrapper to get a value for an operation - const auto get_value = [&val](const std::string & op, - const std::string & member, - bool string_type) -> basic_json& - { - // find value - auto it = val.m_value.object->find(member); - - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; - - // check if desired value is present - if (it == val.m_value.object->end()) - { - throw std::invalid_argument(error_msg + " must have member '" + member + "'"); - } - - // check if result is of type string - if (string_type and not it->second.is_string()) - { - throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); - } - - // no error: return value - return it->second; - }; - - // type check - if (not val.is_object()) - { - throw std::invalid_argument("JSON patch must be an array of objects"); - } - - // collect mandatory members - const std::string op = get_value("op", "op", true); - const std::string path = get_value(op, "path", true); - json_pointer ptr(path); - - switch (get_op(op)) - { - case patch_operations::add: - { - operation_add(ptr, get_value("add", "value", false)); - break; - } - - case patch_operations::remove: - { - operation_remove(ptr); - break; - } - - case patch_operations::replace: - { - // the "path" location must exist - use at() - result.at(ptr) = get_value("replace", "value", false); - break; - } - - case patch_operations::move: - { - const std::string from_path = get_value("move", "from", true); - json_pointer from_ptr(from_path); - - // the "from" location must exist - use at() - basic_json v = result.at(from_ptr); - - // The move operation is functionally identical to a - // "remove" operation on the "from" location, followed - // immediately by an "add" operation at the target - // location with the value that was just removed. - operation_remove(from_ptr); - operation_add(ptr, v); - break; - } - - case patch_operations::copy: - { - const std::string from_path = get_value("copy", "from", true);; - const json_pointer from_ptr(from_path); - - // the "from" location must exist - use at() - result[ptr] = result.at(from_ptr); - break; - } - - case patch_operations::test: - { - bool success = false; - try - { - // check if "value" matches the one at "path" - // the "path" location must exist - use at() - success = (result.at(ptr) == get_value("test", "value", false)); - } - catch (std::out_of_range&) - { - // ignore out of range errors: success remains false - } - - // throw an exception if test fails - if (not success) - { - throw std::domain_error("unsuccessful: " + val.dump()); - } - - break; - } - - case patch_operations::invalid: - { - // op must be "add", "remove", "replace", "move", "copy", or - // "test" - throw std::invalid_argument("operation value '" + op + "' is invalid"); - } - } - } - - return result; - } - - /*! - @brief creates a diff as a JSON patch - - Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can - be changed into the value @a target by calling @ref patch function. - - @invariant For two JSON values @a source and @a target, the following code - yields always `true`: - @code {.cpp} - source.patch(diff(source, target)) == target; - @endcode - - @note Currently, only `remove`, `add`, and `replace` operations are - generated. - - @param[in] source JSON value to copare from - @param[in] target JSON value to copare against - @param[in] path helper value to create JSON pointers - - @return a JSON patch to convert the @a source to @a target - - @complexity Linear in the lengths of @a source and @a target. - - @liveexample{The following code shows how a JSON patch is created as a - diff for two JSON values.,diff} - - @sa @ref patch -- apply a JSON patch - - @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) - - @since version 2.0.0 - */ - static basic_json diff(const basic_json& source, - const basic_json& target, - std::string path = "") - { - // the patch - basic_json result(value_t::array); - - // if the values are the same, return empty patch - if (source == target) - { - return result; - } - - if (source.type() != target.type()) - { - // different types: replace value - result.push_back( - { - {"op", "replace"}, - {"path", path}, - {"value", target} - }); - } - else - { - switch (source.type()) - { - case value_t::array: - { - // first pass: traverse common elements - size_t i = 0; - while (i < source.size() and i < target.size()) - { - // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - ++i; - } - - // i now reached the end of at least one array - // in a second pass, traverse the remaining elements - - // remove my remaining elements - const auto end_index = static_cast(result.size()); - while (i < source.size()) - { - // add operations in reverse order to avoid invalid - // indices - result.insert(result.begin() + end_index, object( - { - {"op", "remove"}, - {"path", path + "/" + std::to_string(i)} - })); - ++i; - } - - // add other remaining elements - while (i < target.size()) - { - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + std::to_string(i)}, - {"value", target[i]} - }); - ++i; - } - - break; - } - - case value_t::object: - { - // first pass: traverse this object's elements - for (auto it = source.begin(); it != source.end(); ++it) - { - // escape the key name to be used in a JSON patch - const auto key = json_pointer::escape(it.key()); - - if (target.find(it.key()) != target.end()) - { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - } - else - { - // found a key that is not in o -> remove it - result.push_back(object( - { - {"op", "remove"}, - {"path", path + "/" + key} - })); - } - } - - // second pass: traverse other object's elements - for (auto it = target.begin(); it != target.end(); ++it) - { - if (source.find(it.key()) == source.end()) - { - // found a key that is not in this -> add it - const auto key = json_pointer::escape(it.key()); - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + key}, - {"value", it.value()} - }); - } - } - - break; - } - - default: - { - // both primitive type: replace value - result.push_back( - { - {"op", "replace"}, - {"path", path}, - {"value", target} - }); - break; - } - } - } - - return result; - } - - /// @} -}; - - -///////////// -// presets // -///////////// - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; -} - - -/////////////////////// -// nonmember support // -/////////////////////// - -// specialization of std::swap, and std::hash -namespace std -{ -/*! -@brief exchanges the values of two JSON objects - -@since version 1.0.0 -*/ -template <> -inline void swap(nlohmann::json& j1, - nlohmann::json& j2) noexcept( - is_nothrow_move_constructible::value and - is_nothrow_move_assignable::value - ) -{ - j1.swap(j2); -} - -/// hash value for JSON objects -template <> -struct hash -{ - /*! - @brief return a hash value for a JSON object - - @since version 1.0.0 - */ - std::size_t operator()(const nlohmann::json& j) const - { - // a naive hashing via the string representation - const auto& h = hash(); - return h(j.dump()); - } -}; -} - -/*! -@brief user-defined string literal for JSON values - -This operator implements a user-defined string literal for JSON objects. It -can be used by adding `"_json"` to a string literal and returns a JSON object -if no parse error occurred. - -@param[in] s a string representation of a JSON object -@return a JSON object - -@since version 1.0.0 -*/ -inline nlohmann::json operator "" _json(const char* s, std::size_t) -{ - return nlohmann::json::parse(reinterpret_cast(s)); -} - -/*! -@brief user-defined string literal for JSON pointer - -This operator implements a user-defined string literal for JSON Pointers. It -can be used by adding `"_json"` to a string literal and returns a JSON pointer -object if no parse error occurred. - -@param[in] s a string representation of a JSON Pointer -@return a JSON pointer object - -@since version 2.0.0 -*/ -inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t) -{ - return nlohmann::json::json_pointer(s); -} - -// restore GCC/clang diagnostic settings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic pop -#endif - -#endif diff --git a/ext/offbase/offbase.hpp b/ext/offbase/offbase.hpp deleted file mode 100644 index acce8c4a..00000000 --- a/ext/offbase/offbase.hpp +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Offbase: a super-minimal in-filesystem JSON object persistence store - */ - -#ifndef OFFBASE_HPP__ -#define OFFBASE_HPP__ - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include "json/json.hpp" - -#define OFFBASE_PATH_SEP "/" - -/** - * A super-minimal in-filesystem JSON object persistence store - */ -class offbase : public nlohmann::json -{ -public: - offbase(const char *p) : - nlohmann::json(nlohmann::json::object()), - _path(p), - _saved(nlohmann::json::object()) - { - this->load(); - } - - ~offbase() - { - this->commit(); - } - - /** - * Load this instance from disk, clearing any existing contents first - * - * If the 'errors' vector is NULL, false is returned and reading aborts - * on any error. If this parameter is non-NULL the paths of errors will - * be added to the vector and reading will continue. False will only be - * returned on really big errors like no path being defined. - * - * @param errors If specified, fill this vector with the paths to any objects that fail read - * @return True on success, false on fatal error - */ - inline bool load(std::vector *errors = (std::vector *)0) - { - if (!_path.length()) - return false; - *this = nlohmann::json::object(); - if (!_loadObj(_path,*this,errors)) - return false; - _saved = *(reinterpret_cast(this)); - return true; - } - - /** - * Commit any pending changes to this object to disk - * - * @return True on success or false if an I/O error occurred - */ - inline bool commit(std::vector *errors = (std::vector *)0) - { - if (!_path.length()) - return false; - if (!_commitObj(_path,*this,&_saved,errors)) - return false; - _saved = *(reinterpret_cast(this)); - return true; - } - - static inline std::string escapeKey(const std::string &k) - { - std::string e; - const char *ptr = k.data(); - const char *eof = ptr + k.length(); - char tmp[8]; - while (ptr != eof) { - if ( ((*ptr >= 'a')&&(*ptr <= 'z')) || ((*ptr >= 'A')&&(*ptr <= 'Z')) || ((*ptr >= '0')&&(*ptr <= '9')) || (*ptr == '.') || (*ptr == '_') || (*ptr == '-') || (*ptr == ',') ) - e.push_back(*ptr); - else { - snprintf(tmp,sizeof(tmp),"~%.2x",(unsigned int)*ptr); - e.append(tmp); - } - ++ptr; - } - return e; - } - - static inline std::string unescapeKey(const std::string &k) - { - std::string u; - const char *ptr = k.data(); - const char *eof = ptr + k.length(); - char tmp[8]; - while (ptr != eof) { - if (*ptr == '~') { - if (++ptr == eof) break; - tmp[0] = *ptr; - if (++ptr == eof) break; - tmp[1] = *(ptr++); - tmp[2] = (char)0; - u.push_back((char)strtol(tmp,(char **)0,16)); - } else { - u.push_back(*(ptr++)); - } - } - return u; - } - -private: - static inline bool _readFile(const char *path,std::string &buf) - { - char tmp[4096]; - FILE *f = fopen(path,"rb"); - if (f) { - for(;;) { - long n = (long)fread(tmp,1,sizeof(tmp),f); - if (n > 0) - buf.append(tmp,n); - else break; - } - fclose(f); - return true; - } - return false; - } - - static inline bool _loadArr(const std::string &path,nlohmann::json &arr,std::vector *errors) - { - std::map atmp; // place into an ordered container first because filesystem does not guarantee this - - struct dirent dbuf; - struct dirent *de; - DIR *d = opendir(path.c_str()); - if (d) { - while (!readdir_d(d,&dbuf,&de)) { - if (!de) break; - const std::string name(de->d_name); - if (name.length() != 12) continue; // array entries are XXXXXXXXXX.T - if (name[name.length()-2] == '.') { - if (name[name.length()-1] == 'V') { - std::string buf; - if (_readFile((path + OFFBASE_PATH_SEP + name).c_str(),buf)) { - try { - atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::parse(buf); - } catch ( ... ) { - if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else { - return false; - } - } - } else if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else return false; - } else if (name[name.length()-1] == 'O') { - if (!_loadObj(path + OFFBASE_PATH_SEP + name,atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::object(),errors)) - return false; - } else if (name[name.length()-1] == 'A') { - if (!_loadArr(path + OFFBASE_PATH_SEP + name,atmp[strtoul(name.substr(0,10).c_str(),(char **)0,16)] = nlohmann::json::array(),errors)) - return false; - } - } - } - closedir(d); - } else if (errors) { - errors->push_back(path); - } else return false; - - if (atmp.size() > 0) { - unsigned long lasti = 0; - for(std::map::iterator i(atmp.begin());i!=atmp.end();++i) { - for(unsigned long k=lasti;kfirst;++k) // fill any gaps with nulls - arr.push_back(nlohmann::json(std::nullptr_t)); - lasti = i->first; - arr.push_back(i->second); - } - } - - return true; - } - - static inline bool _loadObj(const std::string &path,nlohmann::json &obj,std::vector *errors) - { - struct dirent dbuf; - struct dirent *de; - DIR *d = opendir(path.c_str()); - if (d) { - while (!readdir_d(d,&dbuf,&de)) { - if (!de) break; - if ((strcmp(de->d_name,".") == 0)||(strcmp(de->d_name,"..") == 0)) continue; // sanity check - const std::string name(de->d_name); - if (name.length() <= 2) continue; - if (name[name.length()-2] == '.') { - if (name[name.length()-1] == 'V') { - std::string buf; - if (_readFile((path + OFFBASE_PATH_SEP + name).c_str(),buf)) { - try { - obj[unescapeKey(name)] = nlohmann::json::parse(buf); - } catch ( ... ) { - if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else { - return false; - } - } - } else if (errors) { - errors->push_back(path + OFFBASE_PATH_SEP + name); - } else return false; - } else if (name[name.length()-1] == 'O') { - if (!_loadObj(path + OFFBASE_PATH_SEP + name,obj[unescapeKey(name)] = nlohmann::json::object(),errors)) - return false; - } else if (name[name.length()-1] == 'A') { - if (!_loadArr(path + OFFBASE_PATH_SEP + name,obj[unescapeKey(name)] = nlohmann::json::array(),errors)) - return false; - } - } - } - closedir(d); - } else if (errors) { - errors->push_back(path); - } else return false; - return true; - } - - static inline void _rmDashRf(const std::string &path) - { - struct dirent dbuf; - struct dirent *de; - DIR *d = opendir(path.c_str()); - if (d) { - while (!readdir_r(d,&dbuf,&de)) { - if (!de) break; - if ((strcmp(de->d_name,".") == 0)||(strcmp(de->d_name,"..") == 0)) continue; // sanity check - const std::string full(path + OFFBASE_PATH_SEP + de->d_name); - if (unlink(full.c_str())) { - _rmDashRf(full); - rmdir(full.c_str()); - } - } - closedir(d); - } - rmdir(path.c_str()); - } - - static inline bool _commitArr(const std::string &path,const nlohmann::json &arr,const nlohmann::json *previous,std::vector *errors) - { - char tmp[32]; - - if (!arr.is_array()) - return false; - - mkdir(path.c_str(),0755); - - for(unsigned long i=0;i<(unsigned long)arr.size();++i) { - const nlohmann::json &value = arr[i]; - - const nlohmann::json *next = (const nlohmann::json *)0; - if ((previous)&&(previous->is_array())&&(i < previous->size())) { - next = &((*previous)[i]); - if (*next == value) - continue; - } - - if (value.is_object()) { - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - if (!_commitObj(path + tmp,value,next,errors)) - return false; - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - unlink((path + tmp).c_str()); - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - } else if (value.is_array()) { - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - if (!_commitArr(path + tmp,value,next,errors)) - return false; - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - unlink((path + tmp).c_str()); - } else { - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - FILE *f = fopen((path + tmp).c_str(),"w"); - if (f) { - const std::string v(value.dump()); - if (fwrite(v.c_str(),v.length(),1,f) != 1) { - fclose(f); - return false; - } else { - fclose(f); - } - } else { - return false; - } - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - } - } - - if ((previous)&&(previous->is_array())) { - for(unsigned long i=(unsigned long)arr.size();i<(unsigned long)previous->size();++i) { - snprintf(tmp,sizeof(tmp),"%s%.10lx.V",OFFBASE_PATH_SEP,i); - unlink((path + tmp).c_str()); - snprintf(tmp,sizeof(tmp),"%s%.10lx.A",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - snprintf(tmp,sizeof(tmp),"%s%.10lx.O",OFFBASE_PATH_SEP,i); - _rmDashRf(path + tmp); - } - } - - return true; - } - - static inline bool _commitObj(const std::string &path,const nlohmann::json &obj,const nlohmann::json *previous,std::vector *errors) - { - if (!obj.is_object()) - return false; - - mkdir(path.c_str(),0755); - - for(nlohmann::json::const_iterator i(obj.begin());i!=obj.end();++i) { - if (i.key().length() == 0) - continue; - - const nlohmann::json *next = (const nlohmann::json *)0; - if ((previous)&&(previous->is_object())) { - nlohmann::json::const_iterator saved(previous->find(i.key())); - if (saved != previous->end()) { - next = &(saved.value()); - if (i.value() == *next) - continue; - } - } - - const std::string keyp(path + OFFBASE_PATH_SEP + escapeKey(i.key())); - if (i.value().is_object()) { - if (!_commitObj(keyp + ".O",i.value(),next,errors)) - return false; - unlink((keyp + ".V").c_str()); - _rmDashRf(keyp + ".A"); - } else if (i.value().is_array()) { - if (!_commitArr(keyp + ".A",i.value(),next,errors)) - return false; - unlink((keyp + ".V").c_str()); - _rmDashRf(keyp + ".O"); - } else { - FILE *f = fopen((keyp + ".V").c_str(),"w"); - if (f) { - const std::string v(i.value().dump()); - if (fwrite(v.c_str(),v.length(),1,f) != 1) { - fclose(f); - return false; - } else { - fclose(f); - } - } else { - return false; - } - _rmDashRf(keyp + ".A"); - _rmDashRf(keyp + ".O"); - } - } - - if ((previous)&&(previous->is_object())) { - for(nlohmann::json::const_iterator i(previous->begin());i!=previous->end();++i) { - if ((i.key().length() > 0)&&(obj.find(i.key()) == obj.end())) { - const std::string keyp(path + OFFBASE_PATH_SEP + escapeKey(i.key())); - unlink((keyp + ".V").c_str()); - _rmDashRf(keyp + ".A"); - _rmDashRf(keyp + ".O"); - } - } - } - - return true; - } - - std::string _path; - nlohmann::json _saved; -}; - -#endif diff --git a/make-mac.mk b/make-mac.mk index e821c4cf..09e04eab 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -65,7 +65,7 @@ else STRIP=strip endif -CXXFLAGS=$(CFLAGS) -fno-rtti +CXXFLAGS=$(CFLAGS) -fno-rtti -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ all: one @@ -78,7 +78,7 @@ one: $(OBJS) service/OneService.o one.o $(CODESIGN) -vvv zerotier-one cli: FORCE - $(CXX) -Os -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl + $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl $(STRIP) zerotier selftest: $(OBJS) selftest.o diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 3f6b9be6..12446909 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -113,7 +113,7 @@ void InetAddress::set(const std::string &ip,unsigned int port) sin6->sin6_port = Utils::hton((uint16_t)port); if (inet_pton(AF_INET6,ip.c_str(),(void *)&(sin6->sin6_addr.s6_addr)) <= 0) memset(this,0,sizeof(InetAddress)); - } else { + } else if (ip.find('.') != std::string::npos) { struct sockaddr_in *sin = reinterpret_cast(this); ss_family = AF_INET; sin->sin_port = Utils::hton((uint16_t)port); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 3a04308b..086bb269 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -107,6 +107,86 @@ std::vector OSUtils::listDirectory(const char *path) return r; } +std::vector OSUtils::listSubdirectories(const char *path) +{ + std::vector r; + +#ifdef __WINDOWS__ + HANDLE hFind; + WIN32_FIND_DATAA ffd; + if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { + do { + if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)) + r.push_back(std::string(ffd.cFileName)); + } while (FindNextFileA(hFind,&ffd)); + FindClose(hFind); + } +#else + struct dirent de; + struct dirent *dptr; + DIR *d = opendir(path); + if (!d) + return r; + dptr = (struct dirent *)0; + for(;;) { + if (readdir_r(d,&de,&dptr)) + break; + if (dptr) { + if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_DIR)) + r.push_back(std::string(dptr->d_name)); + } else break; + } + closedir(d); +#endif + + return r; +} + +bool OSUtils::rmDashRf(const char *path) +{ +#ifdef __WINDOWS__ + HANDLE hFind; + WIN32_FIND_DATAA ffd; + if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { + do { + if ((strcmp(ffd.cFileName,".") != 0)&&(strcmp(ffd.cFileName,"..") != 0)) { + if ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) { + if (DeleteFileA((std::string(path) + ZT_PATH_SEPARATOR_S + ffd.cFileName).c_str()) == FALSE) + return false; + } else { + if (!rmDashRf((std::string(path) + ZT_PATH_SEPARATOR_S + ffd.cFileName).c_str())) + return false; + } + } + } while (FindNextFileA(hFind,&ffd)); + FindClose(hFind); + } + return (RemoveDirectoryA(path) != FALSE); +#else + struct dirent de; + struct dirent *dptr; + DIR *d = opendir(path); + if (!d) + return true; + dptr = (struct dirent *)0; + for(;;) { + if (readdir_r(d,&de,&dptr)) + break; + if ((dptr)&&(strcmp(dptr->d_name,".") != 0)&&(strcmp(dptr->d_name,"..") != 0)) { + std::string p(path); + p.push_back(ZT_PATH_SEPARATOR); + p.append(dptr->d_name); + if (unlink(p.c_str()) != 0) { + if (!rmDashRf(p.c_str())) + return false; + } + } else break; + } + closedir(d); + return (rmdir(path) == 0); +#endif +} + void OSUtils::lockDownFile(const char *path,bool isDir) { #ifdef __UNIX_LIKE__ diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 25bed9fe..4f74344f 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -105,10 +105,26 @@ public: * This returns only files, not sub-directories. * * @param path Path to list - * @return Names of files in directory + * @return Names of files in directory (without path prepended) */ static std::vector listDirectory(const char *path); + /** + * List a directory's subdirectories + * + * @param path Path to list + * @return Names of subdirectories (without path prepended) + */ + static std::vector listSubdirectories(const char *path); + + /** + * Delete a directory and all its files and subdirectories recursively + * + * @param path Path to delete + * @return True on success + */ + static bool rmDashRf(const char *path); + /** * Set modes on a file to something secure * -- cgit v1.2.3 From b72847d50404f4d751184d9977e7bba23050a797 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 13:41:45 -0700 Subject: Finally implement network join auth tokens, at least at the protocol level. --- controller/EmbeddedNetworkController.cpp | 53 ++++++++++++++++++++++++++++++-- node/NetworkConfig.hpp | 3 ++ 2 files changed, 54 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d3f44fb4..dfb93b01 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -658,11 +658,39 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // Stop if network is private and member is not authorized if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + bool authenticatedViaToken = false; + char atok[256]; + if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { + atok[255] = (char)0; // not necessary but YDIFLO + if (strlen(atok) > 0) { // extra sanity check + auto authTokens = network["authTokens"]; + if (authTokens.is_array()) { + for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { + authenticatedViaToken = true; + break; + } + } + } + } + } + } + + if (!authenticatedViaToken) { + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } } // Else compose and send network config + // If we made it here for some reason other than authorized being true, such as this + // being a public network or via a bearer token, then we set this in the member config. + member["authorized"] = true; + nc.networkId = nwid; nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; @@ -1308,6 +1336,26 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["rules"] = nrules; } } + + if (b.count("authTokens")) { + auto authTokens = b["authTokens"]; + if (authTokens.is_array()) { + json nat = json::array(); + for(unsigned long i=0;i 0) { + json t = json::object(); + t["token"] = tstr; + t["expires"] = token.value("expires",0ULL); + nat.push_back(t); + } + } + } + network["authTokens"] = nat; + } + } } catch ( ... ) { return 400; } @@ -1319,6 +1367,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!network.count("v4AssignMode")) network["v4AssignMode"] = "{\"zt\":false}"_json; if (!network.count("v6AssignMode")) network["v6AssignMode"] = "{\"rfc4193\":false,\"zt\":false,\"6plane\":false}"_json; if (!network.count("activeBridges")) network["activeBridges"] = json::array(); + if (!network.count("authTokens")) network["authTokens"] = json::array(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 18244ec9..25e0ccaf 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -101,6 +101,9 @@ namespace ZeroTier { // Maximum number of tags this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS "mt" +// Network join authorization token (if any) +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN "atok" + // These dictionary keys are short so they don't take up much room. // By convention we use upper case for binary blobs, but it doesn't really matter. -- cgit v1.2.3 From faa9a06bf5302b246805ead12690b38c3036d802 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 17:37:37 -0700 Subject: Controller fixes... --- controller/EmbeddedNetworkController.cpp | 219 +++++++++++++++++++------------ controller/README.md | 2 +- node/IncomingPacket.cpp | 6 +- service/OneService.cpp | 4 + 4 files changed, 148 insertions(+), 83 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 30072f95..649ff094 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -60,6 +60,50 @@ using json = nlohmann::json; namespace ZeroTier { +static uint64_t _jI(const json &jv,const uint64_t dfl) +{ + if (jv.is_number()) { + return (uint64_t)jv; + } else if (jv.is_string()) { + std::string s = jv; + return Utils::strToU64(s.c_str()); + } else if (jv.is_boolean()) { + return ((bool)jv ? 1ULL : 0ULL); + } + return dfl; +} +static bool _jB(const json &jv,const bool dfl) +{ + if (jv.is_boolean()) { + return (bool)jv; + } else if (jv.is_number()) { + return ((uint64_t)jv > 0ULL); + } else if (jv.is_string()) { + std::string s = jv; + if (s.length() > 0) { + switch(s[0]) { + case 't': + case 'T': + case '1': + return true; + } + } + return false; + } + return dfl; +} +static std::string _jS(const json &jv,const char *dfl) +{ + if (jv.is_string()) { + return jv; + } else if (jv.is_number()) { + return jv; + } else if (jv.is_boolean()) { + return ((bool)jv ? std::string("1") : std::string("0")); + } + return std::string((dfl) ? dfl : ""); +} + static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; @@ -190,7 +234,7 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return false; std::string t = r["type"]; memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); - if (r.value("not",false)) + if (_jB(r["not"],false)) rule.t = 0x80; else rule.t = 0x00; if (t == "ACTION_DROP") { @@ -201,91 +245,91 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; return true; } else if (t == "MATCH_VLAN_ID") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; - rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); + rule.v.vlanId = (uint16_t)(_jI(r["vlanId"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_VLAN_PCP") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; - rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); + rule.v.vlanPcp = (uint8_t)(_jI(r["vlanPcp"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_VLAN_DEI") { rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; - rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); + rule.v.vlanDei = (uint8_t)(_jI(r["vlanDei"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_ETHERTYPE") { rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); + rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_MAC_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; - const std::string mac(r.value("mac","0")); + const std::string mac(_jS(r["mac"],"0")); Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); return true; } else if (t == "MATCH_MAC_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; - const std::string mac(r.value("mac","0")); + const std::string mac(_jS(r["mac"],"0")); Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); return true; } else if (t == "MATCH_IPV4_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(r.value("ip","0.0.0.0")); + InetAddress ip(_jS(r["ip"],"0.0.0.0")); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV4_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(r.value("ip","0.0.0.0")); + InetAddress ip(_jS(r["ip"],"0.0.0.0")); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV6_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(r.value("ip","::0")); + InetAddress ip(_jS(r["ip"],"::0")); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IPV6_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(r.value("ip","::0")); + InetAddress ip(_jS(r["ip"],"::0")); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IP_TOS") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); + rule.v.ipTos = (uint8_t)(_jI(r["ipTos"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_PROTOCOL") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; - rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); + rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.port[0]) & 0xffffULL); + rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_IP_DEST_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.port[0]) & 0xffffULL); + rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.port[0]) & 0xffffULL); return true; } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; @@ -310,28 +354,28 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_FRAME_SIZE_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(r.value("end",(uint64_t)rule.v.frameSize[0]) & 0xffffULL); + rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; } else if (t == "MATCH_TAGS_SAMENESS") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(r.value("id",0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(r.value("value",0ULL) & 0xffffffffULL); + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } return false; @@ -613,7 +657,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( json member(_readJson(memberJP)); { - std::string haveIdStr = member.value("identity",""); + std::string haveIdStr(_jS(member["identity"],"")); if (haveIdStr.length() > 0) { try { if (Identity(haveIdStr.c_str()) != identity) @@ -630,13 +674,18 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["id"] = identity.address().toString(); member["address"] = member["id"]; member["nwid"] = network["id"]; - member["memberRevision"] = member.value("memberRevision",0ULL) + 1; + member["lastModified"] = now; + { + auto revj = member["revision"]; + const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + member["revision"] = rev; + } // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; - if (!network.value("private",true)) { + if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; - } else if (member.value("authorized",false)) { + } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else { char atok[256]; @@ -648,8 +697,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { authorizedBy = "token"; break; @@ -700,14 +749,14 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If we made it this far, they are authorized. nc.networkId = nwid; - nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; - nc.revision = network.value("revision",0ULL); + nc.revision = _jI(network["revision"],0ULL); nc.issuedTo = identity.address(); - if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); - nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); + if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (_jB(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); + nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); bool amActiveBridge = false; { @@ -732,11 +781,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( auto rules = network["rules"]; if (v6AssignMode.is_object()) { - if ((v6AssignMode.value("rfc4193",false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if ((v6AssignMode.value("6plane",false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } @@ -757,8 +806,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if (nc.routeCount >= ZT_MAX_NETWORK_ROUTES) break; auto route = routes[i]; - InetAddress t(route.value("target","")); - InetAddress v(route.value("via","")); + InetAddress t(_jS(route["target"],"")); + InetAddress v(_jS(route["via"],"")); if ((t)&&(v)&&(t.ss_family == v.ss_family)) { ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); *(reinterpret_cast(&(r->target))) = t; @@ -803,13 +852,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( std::set allocatedIps; bool allocatedIpsLoaded = false; - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); @@ -921,7 +970,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - if (network.value("private",true)) { + if (_jB(network["private"],true)) { CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; @@ -983,8 +1032,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); responseBody.append("\":"); - const std::string rc = member.value("memberRevision","0"); - responseBody.append(rc); + auto rev = member["revision"]; + if (rev.is_number()) + responseBody.append(rev); + else responseBody.push_back('0'); } } } @@ -1006,7 +1057,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( auto recentLog = member["recentLog"]; if ((recentLog.is_array())&&(recentLog.size() > 0)) { auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + if ((mostRecentLog.is_object())&&(_jI(mostRecentLog["ts"],0ULL) >= threshold)) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); responseBody.append("\":"); @@ -1116,8 +1167,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json member(_readJson(_memberJP(nwid,Address(address),true))); try { - if (b.count("authorized")) member["authorized"] = b.value("authorized",false); - if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known + if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); + if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known if (b.count("ipAssignments")) { auto ipa = b["ipAssignments"]; @@ -1144,12 +1195,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["id"] = addrs; member["address"] = addrs; // legacy member["nwid"] = nwids; - member["memberRevision"] = member.value("memberRevision",0ULL) + 1; member["objtype"] = "member"; + member["lastModified"] = OSUtils::now(); + { + auto revj = member["revision"]; + const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + member["revision"] = rev; + } _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); - member["clock"] = OSUtils::now(); + member["clock"] = member["lastModified"]; responseBody = member.dump(2); responseContentType = "application/json"; return 200; @@ -1178,7 +1234,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } } } - test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); + test->reportAtEveryHop = (_jB(b["reportAtEveryHop"],true) ? 1 : 0); if (!test->hopCount) { ::free((void *)test); @@ -1226,11 +1282,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json network(_readJson(_networkJP(nwid,true))); try { - if (b.count("name")) network["name"] = b.value("name",""); - if (b.count("private")) network["private"] = b.value("private",true); - if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); - if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); + if (b.count("name")) network["name"] = _jS(b["name"],""); + if (b.count("private")) network["private"] = _jB(b["private"],true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = _jB(b["enableBroadcast"],false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = _jB(b["allowPassiveBridging"],false); + if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); if (b.count("activeBridges")) { auto ab = b["activeBridges"]; @@ -1249,10 +1305,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( auto nv4m = network["v4AssignMode"]; if (!nv4m.is_object()) nv4m = json::object(); if (b["v4AssignMode"].is_string()) { // backward compatibility - nv4m["zt"] = (b.value("v4AssignMode","") == "zt"); + nv4m["zt"] = (_jS(b["v4AssignMode"],"") == "zt"); } else if (b["v4AssignMode"].is_object()) { auto v4m = b["v4AssignMode"]; - if (v4m.count("zt")) nv4m["zt"] = v4m.value("zt",false); + if (v4m.count("zt")) nv4m["zt"] = _jB(v4m["zt"],false); } if (!nv4m.count("zt")) nv4m["zt"] = false; } @@ -1261,7 +1317,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( auto nv6m = network["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (b["v6AssignMode"].is_string()) { // backward compatibility - std::vector v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); + std::vector v6m(Utils::split(_jS(b["v6AssignMode"],"").c_str(),",","","")); std::sort(v6m.begin(),v6m.end()); v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { @@ -1274,9 +1330,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } } else if (b["v6AssignMode"].is_object()) { auto v6m = b["v6AssignMode"]; - if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); - if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); - if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); + if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); + if (v6m.count("zt")) nv6m["rfc4193"] = _jB(v6m["zt"],false); + if (v6m.count("6plane")) nv6m["rfc4193"] = _jB(v6m["6plane"],false); } if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; if (!nv6m.count("zt")) nv6m["zt"] = false; @@ -1289,8 +1345,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i 0) { json t = json::object(); t["token"] = tstr; - t["expires"] = token.value("expires",0ULL); + t["expires"] = _jI(token["expires"],0ULL); nat.push_back(t); } } @@ -1372,8 +1428,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); if (!network.count("name")) network["name"] = ""; if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; - if (!network.count("v4AssignMode")) network["v4AssignMode"] = "{\"zt\":false}"_json; - if (!network.count("v6AssignMode")) network["v6AssignMode"] = "{\"rfc4193\":false,\"zt\":false,\"6plane\":false}"_json; + if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; + if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("activeBridges")) network["activeBridges"] = json::array(); if (!network.count("authTokens")) network["authTokens"] = json::array(); @@ -1387,7 +1443,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - network["revision"] = network.value("revision",0ULL) + 1ULL; + auto rev = network["revision"]; + network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); network["objtype"] = "network"; _writeJson(_networkJP(nwid,true),network); diff --git a/controller/README.md b/controller/README.md index 2c7541ae..0b57dd25 100644 --- a/controller/README.md +++ b/controller/README.md @@ -5,7 +5,7 @@ ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. -Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, or edited in place, though we do not recommend doing the latter while the controller is running. Also take care if editing in place that you do not save corrupted JSON since the controller may then lose data when it attempts to load, modify, and save. +Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. Technically the JSON files under `controller.d` can also be edited in place, but we do not recommend doing this under a running controller since data loss or corruption might result. Also take care to keep JSON values of the correct types or data loss may also result. ### Scalability and Reliability diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 5c9e80f8..e4f09106 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -804,8 +804,12 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.armor(peer->key(),true); RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } + } catch (std::exception &exc) { + fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); + TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what()); } catch ( ... ) { - TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + fprintf(stderr,"WARNING: network config request failed with exception: unknown exception" ZT_EOL_S); + TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } return true; } diff --git a/service/OneService.cpp b/service/OneService.cpp index 74628168..7ce45beb 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1705,7 +1705,11 @@ public: if (_controlPlane) scode = _controlPlane->handleRequest(tc->from,tc->parser.method,tc->url,tc->headers,tc->body,data,contentType); else scode = 500; + } catch (std::exception &exc) { + fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: %s" ZT_EOL_S,exc.what()); + scode = 500; } catch ( ... ) { + fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: unknown exceptino" ZT_EOL_S); scode = 500; } -- cgit v1.2.3 From b0d888d235ad8f830fb38090e35f80d49a84ebbf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 22 Aug 2016 14:25:59 -0700 Subject: Signing of Capability and Tag objects. --- controller/EmbeddedNetworkController.cpp | 36 ++++++++++++++++++++++++++++---- node/Capability.hpp | 3 +-- 2 files changed, 33 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8d7e90c7..1088b852 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -548,8 +548,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) break; - auto rule = rules[i]; - if (_parseRule(rule,nc.rules[nc.ruleCount])) + if (_parseRule(rules[i],nc.rules[nc.ruleCount])) ++nc.ruleCount; } } @@ -559,18 +558,47 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(unsigned long i=0;i 0)) { + ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; + unsigned int caprc = 0; + auto caprj = cap["rules"]; + if ((caprj.is_array())&&(caprj.size() > 0)) { + for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) + break; + if (_parseRule(caprj[j],capr[caprc])) + ++caprc; + } + } + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,1,capr,caprc); + if (nc.capabilities[nc.capabilityCount].sign(signingId,identity.address())) + ++nc.capabilityCount; + if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + break; } } } if (memberTags.is_array()) { + std::map< uint32_t,uint32_t > tagsById; + for(unsigned long i=0;i::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { + if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + break; + nc.tags[nc.tagCount] = Tag(nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,identity.address(),t->first,t->second); + if (nc.tags[nc.tagCount].sign(signingId)) + ++nc.tagCount; + } } if (routes.is_array()) { diff --git a/node/Capability.hpp b/node/Capability.hpp index c129485d..689a2c6a 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -73,12 +73,11 @@ public: * @param nwid Network ID * @param ts Timestamp (at controller) * @param expiration Expiration relative to network config timestamp - * @param name Capability short name (max strlen == ZT_MAX_CAPABILITY_NAME_LENGTH, overflow ignored) * @param mccl Maximum custody chain length (1 to create non-transferrable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,const char *name,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { memset(this,0,sizeof(Capability)); _nwid = nwid; -- cgit v1.2.3 From 9a3c652a518c40050a0190b489af9ab11647b0b0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 22 Aug 2016 18:06:46 -0700 Subject: Get rid of expiration in Capability and Tag and move this to NetworkConfig so it can be set network-wide and reset if needed. Also add NetworkConfig field for this and centralize checking of credential time validity. --- controller/EmbeddedNetworkController.cpp | 4 +-- node/Capability.hpp | 12 +------- node/CertificateOfMembership.hpp | 49 ++++++++------------------------ node/Membership.cpp | 2 +- node/Membership.hpp | 10 +++---- node/Network.cpp | 14 ++++----- node/NetworkConfig.cpp | 2 ++ node/NetworkConfig.hpp | 20 +++++++++++++ node/Tag.hpp | 8 +----- 9 files changed, 51 insertions(+), 70 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1088b852..738fea9a 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -576,7 +576,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( ++caprc; } } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,1,capr,caprc); + nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); if (nc.capabilities[nc.capabilityCount].sign(signingId,identity.address())) ++nc.capabilityCount; if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) @@ -595,7 +595,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(std::map< uint32_t,uint32_t >::const_iterator t(tagsById.begin());t!=tagsById.end();++t) { if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) break; - nc.tags[nc.tagCount] = Tag(nwid,now,now + ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,identity.address(),t->first,t->second); + nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); if (nc.tags[nc.tagCount].sign(signingId)) ++nc.tagCount; } diff --git a/node/Capability.hpp b/node/Capability.hpp index 689a2c6a..b0620891 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -72,17 +72,15 @@ public: * @param id Capability ID * @param nwid Network ID * @param ts Timestamp (at controller) - * @param expiration Expiration relative to network config timestamp * @param mccl Maximum custody chain length (1 to create non-transferrable capability) * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,uint64_t ts,uint64_t expiration,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,uint64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { memset(this,0,sizeof(Capability)); _nwid = nwid; _ts = ts; - _expiration = expiration; _id = id; _maxCustodyChainLength = (mccl > 0) ? ((mccl < ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) ? mccl : (unsigned int)ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH) : 1; _ruleCount = (ruleCount < ZT_MAX_CAPABILITY_RULES) ? ruleCount : ZT_MAX_CAPABILITY_RULES; @@ -110,11 +108,6 @@ public: */ inline uint64_t networkId() const { return _nwid; } - /** - * @return Expiration time relative to network config timestamp - */ - inline uint64_t expiration() const { return _expiration; } - /** * @return Timestamp */ @@ -343,7 +336,6 @@ public: // These are the same between Tag and Capability b.append(_nwid); b.append(_ts); - b.append(_expiration); b.append(_id); b.append((uint16_t)_ruleCount); @@ -381,7 +373,6 @@ public: // These are the same between Tag and Capability _nwid = b.template at(p); p += 8; _ts = b.template at(p); p += 8; - _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; const unsigned int rc = b.template at(p); p += 2; @@ -420,7 +411,6 @@ public: private: uint64_t _nwid; uint64_t _ts; - uint64_t _expiration; uint32_t _id; unsigned int _maxCustodyChainLength; diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index a04f8255..304111d6 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -35,11 +35,6 @@ /** * Default window of time for certificate agreement - * - * Right now we use time for 'revision' so this is the maximum time divergence - * between two certs for them to agree. It comes out to five minutes, which - * gives a lot of margin for error if the controller hiccups or its clock - * drifts but causes de-authorized peers to fall off fast enough. */ #define ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA (ZT_NETWORK_AUTOCONF_DELAY * 5) @@ -93,25 +88,17 @@ public: enum ReservedId { /** - * Revision number of certificate - * - * Certificates may differ in revision number by a designated max - * delta. Differences wider than this cause certificates not to agree. + * Timestamp of certificate */ - COM_RESERVED_ID_REVISION = 0, + COM_RESERVED_ID_TIMESTAMP = 0, /** * Network ID for which certificate was issued - * - * maxDelta here is zero, since this must match. */ COM_RESERVED_ID_NETWORK_ID = 1, /** * ZeroTier address to whom certificate was issued - * - * maxDelta will be 0xffffffffffffffff here since it's permitted to differ - * from peers obviously. */ COM_RESERVED_ID_ISSUED_TO = 2 }; @@ -132,16 +119,16 @@ public: /** * Create from required fields common to all networks * - * @param revision Revision number of certificate + * @param timestamp Timestamp of certificate * @param timestampMaxDelta Maximum variation between timestamps on this net * @param nwid Network ID * @param issuedTo Certificate recipient */ - CertificateOfMembership(uint64_t revision,uint64_t revisionMaxDelta,uint64_t nwid,const Address &issuedTo) + CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo) { - _qualifiers[0].id = COM_RESERVED_ID_REVISION; - _qualifiers[0].value = revision; - _qualifiers[0].maxDelta = revisionMaxDelta; + _qualifiers[0].id = COM_RESERVED_ID_TIMESTAMP; + _qualifiers[0].value = timestamp; + _qualifiers[0].maxDelta = timestampMaxDelta; _qualifiers[1].id = COM_RESERVED_ID_NETWORK_ID; _qualifiers[1].value = nwid; _qualifiers[1].maxDelta = 0; @@ -176,27 +163,15 @@ public: inline operator bool() const throw() { return (_qualifierCount != 0); } /** - * @return Maximum delta for mandatory revision field or 0 if field missing + * @return Timestamp for this cert and maximum delta for timestamp */ - inline uint64_t revisionMaxDelta() const + inline std::pair timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { - if (_qualifiers[i].id == COM_RESERVED_ID_REVISION) - return _qualifiers[i].maxDelta; + if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) + return std::pair(_qualifiers[i].value,_qualifiers[i].maxDelta); } - return 0ULL; - } - - /** - * @return Revision number for this cert - */ - inline uint64_t revision() const - { - for(unsigned int i=0;i<_qualifierCount;++i) { - if (_qualifiers[i].id == COM_RESERVED_ID_REVISION) - return _qualifiers[i].value; - } - return 0ULL; + return std::pair(0ULL,0ULL); } /** diff --git a/node/Membership.cpp b/node/Membership.cpp index dbba7f0d..969032ff 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -86,7 +86,7 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe if (_com == com) return 0; const int vr = com.verify(RR); - if ((vr == 0)&&(com.revision() > _com.revision())) + if ((vr == 0)&&(com.timestamp().first > _com.timestamp().first)) _com = com; return vr; } diff --git a/node/Membership.hpp b/node/Membership.hpp index e9f9d488..92bd7ebf 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -84,10 +84,10 @@ public: { } - inline const Capability *next() + inline const Capability *next(const NetworkConfig &nconf) { while (_i != _e) { - if (_i->second.lastReceived) + if ((_i->second.lastReceived)&&(nconf.isCredentialTimestampValid(_i->second.cap))) return &((_i++)->second.cap); else ++_i; } @@ -137,7 +137,7 @@ public: inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const { const TState *t = _tags.get(id); - return ((t) ? (((t->lastReceived != 0)&&(t->tag.expiration() < nconf.timestamp)) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); + return ((t) ? (((t->lastReceived != 0)&&(nconf.isCredentialTimestampValid(t->tag))) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); } /** @@ -154,7 +154,7 @@ public: TState *ts = (TState *)0; Hashtable::Iterator i(const_cast(this)->_tags); while (i.next(id,ts)) { - if ((ts->lastReceived)&&(ts->tag.expiration() < nconf.timestamp)) { + if ((ts->lastReceived)&&(nconf.isCredentialTimestampValid(ts->tag))) { if (n >= maxTags) return n; ids[n] = *id; @@ -172,7 +172,7 @@ public: inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const { std::map::const_iterator c(_caps.find(id)); - return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(c->second.cap.expiration() < nconf.timestamp)) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); + return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(nconf.isCredentialTimestampValid(c->second.cap))) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); } /** diff --git a/node/Network.cpp b/node/Network.cpp index 0bd4ea55..7adb6aeb 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -102,7 +102,7 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig // 0 == no match, -1 == match/drop, 1 == match/accept static int _doZtFilter( const RuntimeEnvironment *RR, - const uint64_t nwid, + const NetworkConfig &nconf, const bool inbound, const Address &ztSource, const Address &ztDest, @@ -155,7 +155,7 @@ static int _doZtFilter( case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(nwid); + outp.append(nconf.networkId); outp.append((uint8_t)((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02)); macDest.appendTo(outp); macSource.appendTo(outp); @@ -481,7 +481,7 @@ bool Network::filterOutgoingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,_id,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -491,7 +491,7 @@ bool Network::filterOutgoingPacket( for(unsigned int c=0;c<_config.capabilityCount;++c) { relevantLocalTagCount = 0; - switch (_doZtFilter(RR,_id,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -523,7 +523,7 @@ bool Network::filterIncomingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,_id,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -532,9 +532,9 @@ bool Network::filterIncomingPacket( Membership::CapabilityIterator mci(m); const Capability *c; - while ((c = mci.next())) { + while ((c = mci.next(_config))) { relevantLocalTagCount = 0; - switch(_doZtFilter(RR,_id,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index a4fddf40..14ebb209 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -37,6 +37,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,this->credentialTimeToLive)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; @@ -202,6 +203,7 @@ bool NetworkConfig::fromDictionary(const Identity &controllerId,Dictionarytimestamp = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,0); + this->credentialTimeToLive = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,0); this->revision = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REVISION,0); this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0); if (!this->issuedTo) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 25e0ccaf..97e9287a 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -125,6 +125,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_TYPE "t" // text #define ZT_NETWORKCONFIG_DICT_KEY_NAME "n" +// credential time to live in ms +#define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL "cttl" // binary serialized certificate of membership #define ZT_NETWORKCONFIG_DICT_KEY_COM "C" // specialists (binary array of uint64_t) @@ -367,11 +369,24 @@ public: return (Tag *)0; } + /** + * Check whether a capability or tag is expired + * + * @param cred Credential to check -- must have timestamp() accessor method + * @return True if credential is NOT expired + */ + template + inline bool isCredentialTimestampValid(const C &cred) const + { + return ( (cred.timestamp() >= timestamp) || ((timestamp - cred.timestamp()) <= credentialTimeToLive) ); + } + /* inline void dump() const { printf("networkId==%.16llx\n",networkId); printf("timestamp==%llu\n",timestamp); + printf("credentialTimeToLive==%llu\n",credentialTimeToLive); printf("revision==%llu\n",revision); printf("issuedTo==%.10llx\n",issuedTo.toInt()); printf("multicastLimit==%u\n",multicastLimit); @@ -405,6 +420,11 @@ public: */ uint64_t timestamp; + /** + * TTL for capabilities and tags + */ + uint64_t credentialTimeToLive; + /** * Controller-side revision counter for this configuration */ diff --git a/node/Tag.hpp b/node/Tag.hpp index bb019474..14cc3a5d 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -62,15 +62,13 @@ public: /** * @param nwid Network ID * @param ts Timestamp - * @param expiration Tag expiration relative to network config timestamp * @param issuedTo Address to which this tag was issued * @param id Tag ID * @param value Tag value */ - Tag(const uint64_t nwid,const uint64_t ts,const uint64_t expiration,const Address &issuedTo,const uint32_t id,const uint32_t value) : + Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : _nwid(nwid), _ts(ts), - _expiration(expiration), _id(id), _value(value), _issuedTo(issuedTo), @@ -79,7 +77,6 @@ public: } inline uint64_t networkId() const { return _nwid; } - inline uint64_t expiration() const { return _expiration; } inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } @@ -120,7 +117,6 @@ public: // These are the same between Tag and Capability b.append(_nwid); b.append(_ts); - b.append(_expiration); b.append(_id); b.append(_value); @@ -146,7 +142,6 @@ public: // These are the same between Tag and Capability _nwid = b.template at(p); p += 8; _ts = b.template at(p); p += 8; - _expiration = b.template at(p); p += 8; _id = b.template at(p); p += 4; _value = b.template at(p); p += 4; @@ -176,7 +171,6 @@ public: private: uint64_t _nwid; uint64_t _ts; - uint64_t _expiration; uint32_t _id; uint32_t _value; Address _issuedTo; -- cgit v1.2.3 From 77f7dcf40a8dbb252e155abf0b7de4a5615a15e7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 09:39:38 -0700 Subject: Obsolete "test network" removal. --- node/Constants.hpp | 13 ------------ node/Network.cpp | 47 +++++++++++++++++------------------------- node/NetworkConfig.hpp | 56 ++++++-------------------------------------------- 3 files changed, 25 insertions(+), 91 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 489203fe..b9308abd 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -354,19 +354,6 @@ */ #define ZT_SUPPORT_OLD_STYLE_NETCONF 1 -/** - * A test pseudo-network-ID that can be joined - * - * Joining this network ID will result in a network with no IP addressing - * and default parameters. No network configuration master will be consulted - * and instead a static config will be used. This is used in built-in testnet - * scenarios and can also be used for external testing. - * - * This is an impossible real network ID since 0xff is a reserved address - * prefix. - */ -#define ZT_TEST_NETWORK_ID 0xffffffffffffffffULL - /** * Desired buffer size for UDP sockets (used in service and osdep but defined here) */ diff --git a/node/Network.cpp b/node/Network.cpp index 7adb6aeb..2a33321c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -409,33 +409,26 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : char confn[128]; Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id); - if (_id == ZT_TEST_NETWORK_ID) { - applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address())); - - // Save a one-byte CR to persist membership in the test network - RR->node->dataStorePut(confn,"\n",1,false); - } else { - bool gotConf = false; - Dictionary *dconf = new Dictionary(); - NetworkConfig *nconf = new NetworkConfig(); - try { - std::string conf(RR->node->dataStoreGet(confn)); - if (conf.length()) { - dconf->load(conf.c_str()); - if (nconf->fromDictionary(Identity(),*dconf)) { - this->setConfiguration(*nconf,false); - _lastConfigUpdate = 0; // we still want to re-request a new config from the network - gotConf = true; - } + bool gotConf = false; + Dictionary *dconf = new Dictionary(); + NetworkConfig *nconf = new NetworkConfig(); + try { + std::string conf(RR->node->dataStoreGet(confn)); + if (conf.length()) { + dconf->load(conf.c_str()); + if (nconf->fromDictionary(Identity(),*dconf)) { + this->setConfiguration(*nconf,false); + _lastConfigUpdate = 0; // we still want to re-request a new config from the network + gotConf = true; } - } catch ( ... ) {} // ignore invalids, we'll re-request - delete nconf; - delete dconf; - - if (!gotConf) { - // Save a one-byte CR to persist membership while we request a real netconf - RR->node->dataStorePut(confn,"\n",1,false); } + } catch ( ... ) {} // ignore invalids, we'll re-request + delete nconf; + delete dconf; + + if (!gotConf) { + // Save a one-byte CR to persist membership while we request a real netconf + RR->node->dataStorePut(confn,"\n",1,false); } if (!_portInitialized) { @@ -698,9 +691,6 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d void Network::requestConfiguration() { - if (_id == ZT_TEST_NETWORK_ID) // pseudo-network-ID, uses locally generated static config - return; - Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); @@ -711,6 +701,7 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_CAPABILITIES,(uint64_t)ZT_MAX_NETWORK_CAPABILITIES); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES,(uint64_t)ZT_MAX_CAPABILITY_RULES); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); if (controller() == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 97e9287a..0ada4710 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -61,7 +61,7 @@ #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE 0x0000020000000000ULL /** - * An anchor is a device that is willing to be one and has been online/stable for a long time on this network + * Anchors are stable devices on this network that can cache multicast info, etc. */ #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR 0x0000040000000000ULL @@ -74,35 +74,30 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 1024 // Network config version -#define ZT_NETWORKCONFIG_VERSION 6 +#define ZT_NETWORKCONFIG_VERSION 7 // Fields for meta-data sent with network config requests // Network config version #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v" - // Protocol version (see Packet.hpp) #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv" - // Software major, minor, revision #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv" - // Maximum number of rules per network this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES "mr" - // Maximum number of capabilities this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_CAPABILITIES "mc" - // Maximum number of rules per capability this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES "mcr" - // Maximum number of tags this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS "mt" - // Network join authorization token (if any) #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN "atok" +// Network configuration meta-data flags +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS "f" // These dictionary keys are short so they don't take up much room. // By convention we use upper case for binary blobs, but it doesn't really matter. @@ -167,6 +162,8 @@ namespace ZeroTier { // node;IP/port[,node;IP/port] #define ZT_NETWORKCONFIG_DICT_KEY_RELAYS_OLD "rl" +// End legacy fields + /** * Network configuration received from network controller nodes * @@ -176,47 +173,6 @@ namespace ZeroTier { class NetworkConfig { public: - /** - * Create an instance of a NetworkConfig for the test network ID - * - * The test network ID is defined as ZT_TEST_NETWORK_ID. This is a - * "fake" network with no real controller and default options. - * - * @param self This node's ZT address - * @return Configuration for test network ID - */ - static inline NetworkConfig createTestNetworkConfig(const Address &self) - { - NetworkConfig nc; - - nc.networkId = ZT_TEST_NETWORK_ID; - nc.timestamp = 1; - nc.revision = 1; - nc.issuedTo = self; - nc.multicastLimit = ZT_MULTICAST_DEFAULT_LIMIT; - nc.flags = ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - nc.type = ZT_NETWORK_TYPE_PUBLIC; - - nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; - nc.ruleCount = 1; - - Utils::snprintf(nc.name,sizeof(nc.name),"ZT_TEST_NETWORK"); - - // Make up a V4 IP from 'self' in the 10.0.0.0/8 range -- no - // guarantee of uniqueness but collisions are unlikely. - uint32_t ip = (uint32_t)((self.toInt() & 0x00ffffff) | 0x0a000000); // 10.x.x.x - if ((ip & 0x000000ff) == 0x000000ff) ip ^= 0x00000001; // but not ending in .255 - if ((ip & 0x000000ff) == 0x00000000) ip ^= 0x00000001; // or .0 - nc.staticIps[0] = InetAddress(Utils::hton(ip),8); - - // Assign an RFC4193-compliant IPv6 address -- will never collide - nc.staticIps[1] = InetAddress::makeIpv6rfc4193(ZT_TEST_NETWORK_ID,self.toInt()); - - nc.staticIpCount = 2; - - return nc; - } - NetworkConfig() { memset(this,0,sizeof(NetworkConfig)); -- cgit v1.2.3 From 0dfc08b31724fe42ad7dc6253b3b673aec90c838 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 11:29:02 -0700 Subject: Tidy up a few minor protocol things, improve documentation in Packet.hpp. --- node/IncomingPacket.cpp | 71 ++++++++++++++++++++++++++++++++----------------- node/Multicaster.cpp | 38 +++++++++++++++++--------- node/NetworkConfig.hpp | 20 ++++++++++++++ node/Packet.hpp | 59 +++++++++++++++++++--------------------- 4 files changed, 120 insertions(+), 68 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e4f09106..e188784a 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -469,29 +469,40 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - if (payloadLength() == ZT_ADDRESS_LENGTH) { - const Address addr(payload(),ZT_ADDRESS_LENGTH); + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_WHOIS); + outp.append(packetId()); + + unsigned int count = 0; + unsigned int ptr = ZT_PACKET_IDX_PAYLOAD; + while ((ptr + ZT_ADDRESS_LENGTH) <= size()) { + const Address addr(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + ptr += ZT_ADDRESS_LENGTH; + const Identity id(RR->topology->getIdentity(addr)); if (id) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_WHOIS); - outp.append(packetId()); id.serialize(outp,false); - outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + ++count; } else { + // If I am not the root and don't know this identity, ask upstream. Downstream + // peer may re-request in the future and if so we will be able to provide it. + if (!RR->topology->amRoot()) + RR->sw->requestWhois(addr); + #ifdef ZT_ENABLE_CLUSTER + // Distribute WHOIS queries across a cluster if we do not know the ID. + // This may result in duplicate OKs to the querying peer, which is fine. if (RR->cluster) RR->cluster->sendDistributedQuery(*this); #endif - if (!RR->topology->amRoot()) { - RR->sw->requestWhois(addr); - return false; // packet parse will be attempted again if we get a reply from upstream - } } - } else { - TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str()); } + + if (count > 0) { + outp.armor(peer->key(),true); + RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + } + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP); } catch ( ... ) { TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); @@ -836,11 +847,26 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar { try { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); + const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS]; const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); const unsigned int gatherLimit = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); //TRACE("< network(RR->node->network(nwid)); + if (network) + network->addCredential(com); + } + } catch ( ... ) { + TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + } + } + if (gatherLimit) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); @@ -854,6 +880,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } + // If we are a member of a cluster, distribute this GATHER across it #ifdef ZT_ENABLE_CLUSTER if ((RR->cluster)&&(gatheredLocally < gatherLimit)) RR->cluster->sendDistributedQuery(*this); @@ -862,7 +889,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP); } catch ( ... ) { - TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } return true; } @@ -878,7 +905,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // Offset -- size of optional fields added to position of later fields unsigned int offset = 0; - if ((flags & 0x01) != 0) { // deprecated but still used by older peers + if ((flags & 0x01) != 0) { + // This is deprecated but may still be sent by old peers CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); if (com) @@ -1053,7 +1081,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt // Tracks total length of variable length fields, initialized to originator credential length below unsigned int vlf; - // Originator credentials + // Originator credentials -- right now only a network ID for which the originator is controller or is authorized by controller is allowed const unsigned int originatorCredentialLength = vlf = at(ZT_PACKET_IDX_PAYLOAD + 23); uint64_t originatorCredentialNetworkId = 0; if (originatorCredentialLength >= 1) { @@ -1085,15 +1113,10 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt vlf += at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); // Check credentials (signature already verified) - NetworkConfig originatorCredentialNetworkConfig; if (originatorCredentialNetworkId) { - if (Network::controllerFor(originatorCredentialNetworkId) == originatorAddress) { - if (!RR->node->network(originatorCredentialNetworkId)) { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we are not a member of that network",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - return true; - } - } else { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID as credential, is not controller for %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); + SharedPtr network(RR->node->network(originatorCredentialNetworkId)); + if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { + TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); return true; } } else { diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index aeee0a85..a6bff6aa 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -224,25 +224,37 @@ void Multicaster::send( if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) { gs.lastExplicitGather = now; - SharedPtr explicitGatherPeers[2]; - explicitGatherPeers[0] = RR->topology->getBestRoot(); - const Address nwidc(Network::controllerFor(nwid)); - if (nwidc != RR->identity.address()) - explicitGatherPeers[1] = RR->topology->getPeer(nwidc); - for(unsigned int k=0;k<2;++k) { - const SharedPtr &p = explicitGatherPeers[k]; - if (!p) - continue; - //TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str()); - Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER); + + Address explicitGatherPeers[16]; + unsigned int numExplicitGatherPeers = 0; + SharedPtr bestRoot(RR->topology->getBestRoot()); + if (bestRoot) + explicitGatherPeers[numExplicitGatherPeers++] = bestRoot->address(); + explicitGatherPeers[numExplicitGatherPeers++] = Network::controllerFor(nwid); + SharedPtr network(RR->node->network(nwid)); + if (network) { + std::vector
anchors(network->config().anchors()); + for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) { + if (*a != RR->identity.address()) { + explicitGatherPeers[numExplicitGatherPeers++] = *a; + if (numExplicitGatherPeers == 16) + break; + } + } + } + + for(unsigned int k=0;kconfig())&&(network->config().isPrivate())) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0; + Packet outp(explicitGatherPeers[k],RR->identity.address(),Packet::VERB_MULTICAST_GATHER); outp.append(nwid); - outp.append((uint8_t)0x00); + outp.append((uint8_t)((com) ? 0x01 : 0x00)); mg.mac().appendTo(outp); outp.append((uint32_t)mg.adi()); outp.append((uint32_t)gatherLimit); + if (com) + com->serialize(outp); RR->sw->send(outp,true); } - gatherLimit = 0; } gs.txQueue.push_back(OutboundMulticast()); diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 0ada4710..9b12aa0e 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -65,6 +65,11 @@ */ #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR 0x0000040000000000ULL +/** + * Device can send CIRCUIT_TESTs for this network + */ +#define ZT_NETWORKCONFIG_SPECIALIST_TYPE_CIRCUIT_TESTER 0x0000080000000000ULL + namespace ZeroTier { // Dictionary capacity needed for max size network config @@ -273,6 +278,21 @@ public: return false; } + /** + * @param byPeer Address to check + * @return True if this peer is allowed to do circuit tests on this network (controller is always true) + */ + inline bool circuitTestingAllowed(const Address &byPeer) const + { + if (byPeer.toInt() == ((networkId >> 24) & 0xffffffffffULL)) + return true; + for(unsigned int i=0;i * <[...] binary serialized identity (see Identity)> * <[1] destination address type> - * [<[...] destination address>] + * [<[...] destination address to which packet was sent>] * <[8] 64-bit world ID of current world> * <[8] 64-bit timestamp of current world> * @@ -592,20 +593,24 @@ public: /** * Query an identity by address: * <[5] address to look up> + * [<[...] additional addresses to look up> * * OK response payload: * <[...] binary serialized identity> + * [<[...] additional binary serialized identities>] * * If querying a cluster, duplicate OK responses may occasionally occur. - * These should be discarded. + * These must be tolerated, which is easy since they'll have info you + * already have. * - * If the address is not found, no response is generated. WHOIS requests - * will time out much like ARP requests and similar do in L2. + * If the address is not found, no response is generated. The semantics + * of WHOIS is similar to ARP and NDP in that persistent retrying can + * be performed. */ VERB_WHOIS = 0x04, /** - * Meet another node at a given protocol address: + * Relay-mediated NAT traversal or firewall punching initiation: * <[1] flags (unused, currently 0)> * <[5] ZeroTier address of peer that might be found at this address> * <[2] 16-bit protocol address port> @@ -619,15 +624,6 @@ public: * * Upon receipt a peer sends HELLO to establish a direct link. * - * Nodes should implement rate control, limiting the rate at which they - * respond to these packets to prevent their use in DDOS attacks. Nodes - * may also ignore these messages if a peer is not known or is not being - * actively communicated with. - * - * Unfortunately the physical address format in this message pre-dates - * InetAddress's serialization format. :( ZeroTier is four years old and - * yes we've accumulated a tiny bit of cruft here and there. - * * No OK or ERROR is generated. */ VERB_RENDEZVOUS = 0x05, @@ -652,7 +648,6 @@ public: * Full Ethernet frame with MAC addressing and optional fields: * <[8] 64-bit network ID> * <[1] flags> - * [<[...] certificate of network membership (DEPRECATED)>] * <[6] destination MAC or all zero for destination node> * <[6] source MAC or all zero for node of origin> * <[2] 16-bit ethertype> @@ -715,6 +710,9 @@ public: * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may * be pushed at any other time to keep exchanged certificates up to date. * + * COMs and other credentials need not be for the same network, since each + * includes its own network ID and signature. + * * OK/ERROR are not generated. */ VERB_NETWORK_CREDENTIALS = 0x0a, @@ -762,10 +760,10 @@ public: * <[6] MAC address of multicast group being queried> * <[4] 32-bit ADI for multicast group being queried> * <[4] 32-bit requested max number of multicast peers> - * [<[...] network certificate of membership (DEPRECATED)>] + * [<[...] network certificate of membership>] * * Flags: - * 0x01 - COM is attached (DEPRECATED) + * 0x01 - COM is attached * * This message asks a peer for additional known endpoints that have * LIKEd a given multicast group. It's sent when the sender wishes @@ -775,8 +773,8 @@ public: * More than one OK response can occur if the response is broken up across * multiple packets or if querying a clustered node. * - * Send VERB_NETWORK_CREDENTIALS prior to GATHERing if doing so from - * upstream nodes like root servers that are not involved in our network. + * The COM should be included so that upstream nodes that are not + * members of our network can validate our request. * * OK response payload: * <[8] 64-bit network ID> @@ -795,7 +793,6 @@ public: * Multicast frame: * <[8] 64-bit network ID> * <[1] flags> - * [<[...] network certificate of membership (DEPRECATED)>] * [<[4] 32-bit implicit gather limit>] * [<[6] source MAC>] * <[6] destination MAC (multicast address)> @@ -890,7 +887,7 @@ public: * <[...] next hop(s) in path> * * Flags: - * 0x01 - Report back to originator at middle hops + * 0x01 - Report back to originator at all hops * 0x02 - Report back to originator at last hop * * Originator credential types: @@ -948,21 +945,21 @@ public: /** * Circuit test hop report: - * <[8] 64-bit timestamp (from original test)> - * <[8] 64-bit test ID (from original test)> + * <[8] 64-bit timestamp (echoed from original test)> + * <[8] 64-bit test ID (echoed from original test)> * <[8] 64-bit reserved field (set to 0, currently unused)> * <[1] 8-bit vendor ID (set to 0, currently unused)> * <[1] 8-bit reporter protocol version> - * <[1] 8-bit reporter major version> - * <[1] 8-bit reporter minor version> - * <[2] 16-bit reporter revision> - * <[2] 16-bit reporter OS/platform> - * <[2] 16-bit reporter architecture> + * <[1] 8-bit reporter software major version> + * <[1] 8-bit reporter software minor version> + * <[2] 16-bit reporter software revision> + * <[2] 16-bit reporter OS/platform or 0 if not specified> + * <[2] 16-bit reporter architecture or 0 if not specified> * <[2] 16-bit error code (set to 0, currently unused)> * <[8] 64-bit report flags (set to 0, currently unused)> - * <[8] 64-bit source packet ID> - * <[5] upstream ZeroTier address from which test was received> - * <[1] 8-bit source packet hop count (ZeroTier hop count)> + * <[8] 64-bit packet ID of received CIRCUIT_TEST packet> + * <[5] upstream ZeroTier address from which CIRCUIT_TEST was received> + * <[1] 8-bit packet hop count of received CIRCUIT_TEST> * <[...] local wire address on which packet was received> * <[...] remote wire address from which packet was received> * <[2] 16-bit length of additional fields> -- cgit v1.2.3 From 68b4ca9b3181e69108bb4120c1c4230e3d09293b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 11:52:10 -0700 Subject: Cleanup. --- node/Network.cpp | 4 ++-- node/NetworkConfig.cpp | 12 +----------- node/NetworkConfig.hpp | 3 +-- 3 files changed, 4 insertions(+), 15 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 2a33321c..d8e3b07a 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -416,7 +416,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : std::string conf(RR->node->dataStoreGet(confn)); if (conf.length()) { dconf->load(conf.c_str()); - if (nconf->fromDictionary(Identity(),*dconf)) { + if (nconf->fromDictionary(*dconf)) { this->setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; @@ -672,7 +672,7 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d try { Identity controllerId(RR->topology->getIdentity(this->controller())); if (controllerId) { - if (nc->fromDictionary(controllerId,*dict)) { + if (nc->fromDictionary(*dict)) { this->setConfiguration(*nc,true); } else { TRACE("error parsing new config with length %u: deserialization of NetworkConfig failed (certificate error?)",(unsigned int)newConfig.length()); diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 14ebb209..0c9c05ca 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -179,18 +179,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b return true; } -bool NetworkConfig::fromDictionary(const Identity &controllerId,Dictionary &d) +bool NetworkConfig::fromDictionary(const Dictionary &d) { - if ((d.contains(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE))&&(controllerId)) { - // FIXME: right now signature are optional since network configs are only - // accepted directly from the controller and the protocol already guarantees - // the sender. In the future these might be made non-optional once old - // controllers that do not sign are gone and if we ever support peer caching - // of network configs. - if (!d.unwrapAndVerify(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE,controllerId.publicKey())) - return false; - } - Buffer *tmp = new Buffer(); try { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 9b12aa0e..a853d020 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -206,11 +206,10 @@ public: /** * Read this network config from a dictionary * - * @param controllerId Controller identity for verification of any signature or NULL identity to skip * @param d Dictionary (non-const since it might be modified during parse, should not be used after call) * @return True if dictionary was valid and network config successfully initialized */ - bool fromDictionary(const Identity &controllerId,Dictionary &d); + bool fromDictionary(const Dictionary &d); /** * @return True if passive bridging is allowed (experimental) -- cgit v1.2.3 From 70368312039f37d08ba687b07a5caad1c57cd8de Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 11:57:56 -0700 Subject: Sign Dictionary in doNETWORK_CONFIG_REQUEST. --- node/Dictionary.hpp | 2 +- node/Identity.hpp | 13 +++++++++++++ node/IncomingPacket.cpp | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 5d453fd9..eab2b162 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -454,7 +454,7 @@ public: { this->erase(sigKey); C25519::Signature sig(C25519::sign(kp,this->data(),this->sizeBytes())); - this->add(sigKey,sig.data,ZT_C25519_SIGNATURE_LEN); + this->add(sigKey,reinterpret_cast(sig.data),ZT_C25519_SIGNATURE_LEN); } /** diff --git a/node/Identity.hpp b/node/Identity.hpp index ef7f2d77..e4522732 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -287,6 +287,19 @@ public: */ inline const C25519::Public &publicKey() const { return _publicKey; } + /** + * @return C25519 key pair (only returns valid pair if private key is present in this Identity object) + */ + inline const C25519::Pair privateKeyPair() const + { + C25519::Pair pair; + pair.pub = _publicKey; + if (_privateKey) + pair.priv = *_privateKey; + else memset(pair.priv.data,0,ZT_C25519_PRIVATE_KEY_LEN); + return pair; + } + /** * @return True if this identity contains something */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e188784a..139661db 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -749,6 +749,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons Dictionary *dconf = new Dictionary(); try { if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { + dconf->wrapWithSignature(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE,RR->identity.privateKeyPair()); const unsigned int totalSize = dconf->sizeBytes(); unsigned int chunkIndex = 0; while (chunkIndex < totalSize) { -- cgit v1.2.3 From 32fa0617004e80c99b341eb1b4753705b515b53a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 13:02:59 -0700 Subject: Compute credential TTL et al. --- controller/EmbeddedNetworkController.cpp | 44 +++++++++++++++++++++++++++++--- controller/EmbeddedNetworkController.hpp | 8 ++++-- controller/README.md | 3 +++ node/CertificateOfMembership.hpp | 5 ---- node/Membership.hpp | 4 +-- node/NetworkConfig.hpp | 16 ++++++++++++ 6 files changed, 67 insertions(+), 13 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 738fea9a..4db219c9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -413,7 +413,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( lrt = now; } - json network(_readJson(_networkJP(nwid,false))); + const json network(_readJson(_networkJP(nwid,false))); if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; @@ -458,7 +458,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If member already has an authorized field, leave it alone. That way its state is // preserved if the user toggles the network back to private. Otherwise set it to // true by default for new members of public nets. - if (!member.count("authorized")) member["authorized"] = true; + if (!member.count("authorized")) { + member["authorized"] = true; + member["lastAuthorizedTime"] = now; + member["lastAuthorizedBy"] = authorizedBy; + } } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else { @@ -476,6 +480,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if ( ((expires == 0ULL)||(expires > now)) && (tok.length() > 0) && (tok == atok) ) { authorizedBy = "token"; member["authorized"] = true; // tokens actually change member authorization state + member["lastAuthorizedTime"] = now; + member["lastAuthorizedBy"] = authorizedBy; break; } } @@ -517,14 +523,28 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } + // ------------------------------------------------------------------------- // If we made it this far, they are authorized. + // ------------------------------------------------------------------------- _NetworkMemberInfo nmi; _getNetworkMemberInfo(now,nwid,nmi); + // Compute credential TTL. This is the "moving window" for COM agreement and + // the global TTL for Capability and Tag objects. (The same value is used + // for both.) This is computed by reference to the last time we deauthorized + // a member, since within the time period since this event any temporal + // differences are not particularly relevant. + uint64_t credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL; + if (now > nmi.mostRecentDeauthTime) + credentialTtl += (now - nmi.mostRecentDeauthTime); + if (credentialTtl > ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL) + credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL; + nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; + nc.credentialTimeToLive = credentialTtl; nc.revision = _jI(network["revision"],0ULL); nc.issuedTo = identity.address(); if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; @@ -777,7 +797,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } if (_jB(network["private"],true)) { - CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); + CertificateOfMembership com(now,credentialTtl,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; } else { @@ -976,10 +996,24 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _initMember(member); try { - if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known + if (b.count("authorized")) { + const bool newAuth = _jB(b["authorized"],false); + const bool oldAuth = _jB(member["authorized"],false); + if (newAuth != oldAuth) { + if (newAuth) { + member["authorized"] = true; + member["lastAuthorizedTime"] = now; + member["lastAuthorizedBy"] = "user"; + } else { + member["authorized"] = false; + member["lastDeauthorizedTime"] = now; + } + } + } + if (b.count("ipAssignments")) { auto ipa = b["ipAssignments"]; if (ipa.is_array()) { @@ -1476,6 +1510,8 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid nmi.allocatedIps.insert(mip); } } + } else { + nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(nm->second["lastDeauthorizedTime"],0ULL)); } } } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index e6b4bb59..3613e0ef 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -141,12 +141,13 @@ private: // This does lock _networkMemberCache_m struct _NetworkMemberInfo { - _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0) {} + _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} std::set
activeBridges; std::set allocatedIps; unsigned long authorizedMemberCount; unsigned long activeMemberCount; unsigned long totalMemberCount; + uint64_t mostRecentDeauthTime; }; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); @@ -154,7 +155,10 @@ private: inline void _initMember(nlohmann::json &member) { if (!member.count("authorized")) member["authorized"] = false; - if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); + if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; + if (!member.count("lastAuthorizedBy")) member["lastAuthorizedBy"] = ""; + if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; + if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); if (!member.count("activeBridge")) member["activeBridge"] = false; if (!member.count("tags")) member["tags"] = nlohmann::json::array(); diff --git a/controller/README.md b/controller/README.md index 339adb31..189fcdbd 100644 --- a/controller/README.md +++ b/controller/README.md @@ -208,6 +208,9 @@ This returns an object containing all currently online members and the most rece | nwid | string | 16-digit network ID | no | | clock | integer | Current clock, ms since epoch | no | | authorized | boolean | Is member authorized? (for private networks) | YES | +| lastAuthorizedTime | integer | Time 'authorized' was last set to 'true' | no | +| lastAuthorizedBy | string | What last set 'authorized' to 'true'? | no | +| lastDeauthorizedTime | integer | Time 'authorized' was last set to 'false' | no | | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 304111d6..2d7c2cb3 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -33,11 +33,6 @@ #include "Identity.hpp" #include "Utils.hpp" -/** - * Default window of time for certificate agreement - */ -#define ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA (ZT_NETWORK_AUTOCONF_DELAY * 5) - /** * Maximum number of qualifiers allowed in a COM (absolute max: 65535) */ diff --git a/node/Membership.hpp b/node/Membership.hpp index 92bd7ebf..a845b992 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -32,10 +32,10 @@ #include "NetworkConfig.hpp" // Expiration time for capability and tag cache -#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 4) +#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME 600000 // Expiration time for Memberships (used in Peer::clean()) -#define ZT_MEMBERSHIP_EXPIRATION_TIME (ZT_MEMBERSHIP_STATE_EXPIRATION_TIME * 4) +#define ZT_MEMBERSHIP_EXPIRATION_TIME (ZT_MEMBERSHIP_STATE_EXPIRATION_TIME * 2) namespace ZeroTier { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index a853d020..e1a4e302 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -40,6 +40,22 @@ #include "Dictionary.hpp" #include "Identity.hpp" +/** + * Default maximum credential TTL and maxDelta for COM timestamps + * + * The current value is two hours, providing ample time for a controller to + * experience fail-over, etc. + */ +#define ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL 7200000ULL + +/** + * Default minimum credential TTL and maxDelta for COM timestamps + * + * This is just slightly over three minutes and provides three retries for + * all currently online members to refresh. + */ +#define ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL 185000ULL + /** * Flag: allow passive bridging (experimental) */ -- cgit v1.2.3 From 0a7a33ef8fb8cd3cdf25c48bd221298279e690c2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 13:46:36 -0700 Subject: Instantaneous blacklisting and credential revocation. --- node/IncomingPacket.cpp | 23 +++++++++++++------- node/Membership.hpp | 56 ++++++++++++++++++++++++++++++++++++++++++------- node/Network.cpp | 12 ++++------- node/Network.hpp | 11 ++++++++++ node/NetworkConfig.hpp | 12 ----------- node/Packet.hpp | 25 +++++++++++++++------- 6 files changed, 96 insertions(+), 43 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 139661db..0ecc68be 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -829,13 +829,22 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - unsigned int p = ZT_PACKET_IDX_PAYLOAD; - while ((p + 8) <= size()) { - const uint64_t nwid = at(p); p += 8; - if (Network::controllerFor(nwid) == peer->address()) { - SharedPtr network(RR->node->network(nwid)); - if (network) - network->requestConfiguration(); + const uint64_t nwid = at(ZT_PACKET_IDX_PAYLOAD); + + if (Network::controllerFor(nwid) == peer->address()) { + SharedPtr network(RR->node->network(nwid)); + if (network) { + network->requestConfiguration(); + } else { + TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid); + return true; + } + + const unsigned int blacklistCount = at(ZT_PACKET_IDX_PAYLOAD + 8); + unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 10; + for(unsigned int i=0;iblacklistBefore(Address(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH),at(ptr + 5)); + ptr += 13; } } } catch ( ... ) { diff --git a/node/Membership.hpp b/node/Membership.hpp index a845b992..dc525483 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -42,7 +42,10 @@ namespace ZeroTier { class RuntimeEnvironment; /** - * A container for certificates of membership and other credentials for peer participation on networks + * A container for certificates of membership and other network credentials + * + * This is kind of analogous to a join table between Peer and Network. It is + * presently held by the Network object for each participating Peer. */ class Membership { @@ -73,12 +76,13 @@ private: public: /** - * A wrapper to iterate through capabilities in ascending order of capability ID + * A wrapper to iterate through member capabilities in ascending order of capability ID and return only valid ones */ class CapabilityIterator { public: CapabilityIterator(const Membership &m) : + _m(m), _i(m._caps.begin()), _e(m._caps.end()) { @@ -87,7 +91,7 @@ public: inline const Capability *next(const NetworkConfig &nconf) { while (_i != _e) { - if ((_i->second.lastReceived)&&(nconf.isCredentialTimestampValid(_i->second.cap))) + if ((_i->second.lastReceived)&&(_m.isCredentialTimestampValid(nconf,_i->second.cap))) return &((_i++)->second.cap); else ++_i; } @@ -95,12 +99,14 @@ public: } private: + const Membership &_m; std::map::const_iterator _i,_e; }; friend class CapabilityIterator; Membership() : _lastPushedCom(0), + _blacklistBefore(0), _com(), _caps(), _tags(8) @@ -125,9 +131,30 @@ public: bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount); /** - * @return This peer's COM if they have sent one + * @param nconf Our network config + * @return True if this peer is allowed on this network at all + */ + inline bool isAllowedOnNetwork(const NetworkConfig &nconf) const + { + if (nconf.isPublic()) + return true; + if ((_blacklistBefore)&&(_com.timestamp().first <= _blacklistBefore)) + return false; + return nconf.com.agreesWith(_com); + } + + /** + * Check whether a capability or tag is expired + * + * @param cred Credential to check -- must have timestamp() accessor method + * @return True if credential is NOT expired */ - inline const CertificateOfMembership &com() const { return _com; } + template + inline bool isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred) const + { + const uint64_t ts = cred.timestamp(); + return ( ( (ts >= nconf.timestamp) || ((nconf.timestamp - ts) <= nconf.credentialTimeToLive) ) && (ts > _blacklistBefore) ); + } /** * @param nconf Network configuration @@ -137,7 +164,7 @@ public: inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const { const TState *t = _tags.get(id); - return ((t) ? (((t->lastReceived != 0)&&(nconf.isCredentialTimestampValid(t->tag))) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); + return ((t) ? (((t->lastReceived != 0)&&(isCredentialTimestampValid(nconf,t->tag))) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); } /** @@ -154,7 +181,7 @@ public: TState *ts = (TState *)0; Hashtable::Iterator i(const_cast(this)->_tags); while (i.next(id,ts)) { - if ((ts->lastReceived)&&(nconf.isCredentialTimestampValid(ts->tag))) { + if ((ts->lastReceived)&&(isCredentialTimestampValid(nconf,ts->tag))) { if (n >= maxTags) return n; ids[n] = *id; @@ -172,7 +199,7 @@ public: inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const { std::map::const_iterator c(_caps.find(id)); - return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(nconf.isCredentialTimestampValid(c->second.cap))) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); + return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(isCredentialTimestampValid(nconf,c->second.cap))) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); } /** @@ -196,6 +223,16 @@ public: */ int addCredential(const RuntimeEnvironment *RR,const Capability &cap); + /** + * Blacklist COM, tags, and capabilities before this time + * + * @param ts Blacklist cutoff + */ + inline void blacklistBefore(const uint64_t ts) + { + _blacklistBefore = ts; + } + /** * Clean up old or stale entries * @@ -234,6 +271,9 @@ private: // Last time we pushed our COM to this peer uint64_t _lastPushedCom; + // Time before which to blacklist credentials from this peer + uint64_t _blacklistBefore; + // COM from this peer CertificateOfMembership _com; diff --git a/node/Network.cpp b/node/Network.cpp index d8e3b07a..1319df4e 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -888,16 +888,12 @@ bool Network::_isAllowed(const SharedPtr &peer) const // Assumes _lock is locked try { if (_config) { - if (_config.isPublic()) { - return true; - } else { - const Membership *m = _memberships.get(peer->address()); - if (m) - return _config.com.agreesWith(m->com()); - } + const Membership *const m = _memberships.get(peer->address()); + if (m) + return m->isAllowedOnNetwork(_config); } } catch ( ... ) { - TRACE("isAllowed() check failed for peer %s: unexpected exception: unexpected exception",peer->address().toString().c_str()); + TRACE("isAllowed() check failed for peer %s: unexpected exception",peer->address().toString().c_str()); } return false; } diff --git a/node/Network.hpp b/node/Network.hpp index d13918cf..37154dc7 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -387,6 +387,17 @@ public: return _memberships[tag.issuedTo()].addCredential(RR,tag); } + /** + * Blacklist COM, tags, and capabilities before this time + * + * @param ts Blacklist cutoff + */ + inline void blacklistBefore(const Address &peerAddress,const uint64_t ts) + { + Mutex::Lock _l(_lock); + _memberships[peerAddress].blacklistBefore(ts); + } + /** * Destroy this network * diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index e1a4e302..22ffb1cf 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -360,18 +360,6 @@ public: return (Tag *)0; } - /** - * Check whether a capability or tag is expired - * - * @param cred Credential to check -- must have timestamp() accessor method - * @return True if credential is NOT expired - */ - template - inline bool isCredentialTimestampValid(const C &cred) const - { - return ( (cred.timestamp() >= timestamp) || ((timestamp - cred.timestamp()) <= credentialTimeToLive) ); - } - /* inline void dump() const { diff --git a/node/Packet.hpp b/node/Packet.hpp index c2e6da00..fed6aacf 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -742,14 +742,23 @@ public: VERB_NETWORK_CONFIG_REQUEST = 0x0b, /** - * Network configuration refresh request: - * <[...] array of 64-bit network IDs> - * - * This can be sent by the network controller to inform a node that it - * should now make a NETWORK_CONFIG_REQUEST. - * - * It does not generate an OK or ERROR message, and is treated only as - * a hint to refresh now. + * Network configuration update push: + * <[8] network ID to refresh> + * <[2] 16-bit number of address/timestamp pairs to blacklist> + * [<[5] ZeroTier address of peer being revoked>] + * [<[8] blacklist credentials older than this timestamp>] + * [<[...] additional address/timestamp pairs>] + * + * This can be sent by a network controller to both request that a network + * config be updated and push instantaneous revocations of specific peers + * or peer credentials. + * + * Specific revocations can be pushed to blacklist a specific peer's + * credentials (COM, tags, and capabilities) if older than a specified + * timestamp. This can be used to accomplish expedited revocation of + * a peer's access to things on a network or to the network itself among + * those other peers that can currently reach the controller. This is not + * the only mechanism for revocation of course, but it's the fastest. */ VERB_NETWORK_CONFIG_REFRESH = 0x0c, -- cgit v1.2.3 From 0ee4d3554a072863caa59ca2b45122996258617e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 Aug 2016 14:38:20 -0700 Subject: Stub out USER_MESSAGE. --- node/IncomingPacket.cpp | 2 ++ node/Packet.cpp | 1 + node/Packet.hpp | 13 ++++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 0ecc68be..1aecfdb7 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -109,6 +109,8 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); case Packet::VERB_REQUEST_PROOF_OF_WORK: return _doREQUEST_PROOF_OF_WORK(RR,peer); + case Packet::VERB_USER_MESSAGE: + return true; } } else { RR->sw->requestWhois(sourceAddress); diff --git a/node/Packet.cpp b/node/Packet.cpp index aadee00b..9630e5bb 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -47,6 +47,7 @@ const char *Packet::verbString(Verb v) case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST"; case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT"; case VERB_REQUEST_PROOF_OF_WORK: return "REQUEST_PROOF_OF_WORK"; + case VERB_USER_MESSAGE: return "USER_MESSAGE"; } return "(unknown)"; } diff --git a/node/Packet.hpp b/node/Packet.hpp index fed6aacf..0a5d3fec 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1030,7 +1030,18 @@ public: * * ERROR has no payload. */ - VERB_REQUEST_PROOF_OF_WORK = 0x13 + VERB_REQUEST_PROOF_OF_WORK = 0x13, + + /** + * A message with arbitrary user-definable content: + * <[8] 64-bit arbitrary message type ID> + * [<[...] message payload>] + * + * This can be used to send arbitrary messages over VL1. It generates no + * OK or ERROR and has no special semantics outside of whatever the user + * (via the ZeroTier core API) chooses to give it. + */ + VERB_USER_MESSAGE = 0x14 }; /** -- cgit v1.2.3 From 8e3463d47a8e7565784f349f359ebe7f4a4d0e57 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 13:37:57 -0700 Subject: Add length limit to TEE and REDIRECT, and completely factor out old C json-parser to eliminate a dependency. --- AUTHORS.md | 8 +- controller/EmbeddedNetworkController.cpp | 18 +- ext/json-parser/AUTHORS | 20 - ext/json-parser/LICENSE | 26 - ext/json-parser/README.md | 97 --- ext/json-parser/json.c | 1012 ------------------------------ ext/json-parser/json.h | 283 --------- include/ZeroTierOne.h | 14 + make-freebsd.mk | 2 +- make-linux.mk | 6 - make-mac.mk | 2 +- node/Capability.hpp | 9 + node/Network.cpp | 6 +- node/Packet.hpp | 1 + one.cpp | 288 +++------ service/ControlPlane.cpp | 33 +- 16 files changed, 158 insertions(+), 1667 deletions(-) delete mode 100644 ext/json-parser/AUTHORS delete mode 100644 ext/json-parser/LICENSE delete mode 100644 ext/json-parser/README.md delete mode 100644 ext/json-parser/json.c delete mode 100644 ext/json-parser/json.h (limited to 'node') diff --git a/AUTHORS.md b/AUTHORS.md index aa9e9111..90a64ef6 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -39,11 +39,11 @@ These are included in ext/ for platforms that do not have them available in comm * Home page: https://github.com/joyent/http-parser/ * License grant: MIT/Expat - * json-parser by James McLaughlin + * C++11 json (nlohmann/json) by Niels Lohmann - * Files: ext/json-parser/* - * Home page: https://github.com/udp/json-parser/ - * License grant: BSD attribution + * Files: ext/json/* + * Home page: https://github.com/nlohmann/json + * License grant: MIT * TunTapOSX by Mattias Nissler diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 4e8d1aad..0a4a17f8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -121,11 +121,15 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_ACTION_TEE: r["type"] = "ACTION_TEE"; - r["zt"] = Address(rule.v.zt).toString(); + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (uint64_t)rule.v.fwd.flags; + r["length"] = (uint64_t)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; - r["zt"] = Address(rule.v.zt).toString(); + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (uint64_t)rule.v.fwd.flags; + r["length"] = (uint64_t)rule.v.fwd.length; break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; @@ -235,7 +239,7 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) { if (r.is_object()) return false; - std::string t = r["type"]; + const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); if (_jB(r["not"],false)) rule.t = 0x80; @@ -248,11 +252,15 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; diff --git a/ext/json-parser/AUTHORS b/ext/json-parser/AUTHORS deleted file mode 100644 index 6a5c799f..00000000 --- a/ext/json-parser/AUTHORS +++ /dev/null @@ -1,20 +0,0 @@ -All contributors arranged by first commit: - -James McLaughlin -Alex Gartrell -Peter Scott -Mathias Kaerlev -Emiel Mols -Czarek Tomczak -Nicholas Braden -Ivan Kozub -Árpád Goretity -Igor Gnatenko -Haïkel Guémar -Tobias Waldekranz -Patrick Donnelly -Wilmer van der Gaast -Jin Wei -François Cartegnie -Matthijs Boelstra - diff --git a/ext/json-parser/LICENSE b/ext/json-parser/LICENSE deleted file mode 100644 index 1aee375e..00000000 --- a/ext/json-parser/LICENSE +++ /dev/null @@ -1,26 +0,0 @@ - - Copyright (C) 2012, 2013 James McLaughlin et al. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - diff --git a/ext/json-parser/README.md b/ext/json-parser/README.md deleted file mode 100644 index e0b70b6d..00000000 --- a/ext/json-parser/README.md +++ /dev/null @@ -1,97 +0,0 @@ -Very low footprint JSON parser written in portable ANSI C. - -* BSD licensed with no dependencies (i.e. just drop the C file into your project) -* Never recurses or allocates more memory than it needs -* Very simple API with operator sugar for C++ - -[![Build Status](https://secure.travis-ci.org/udp/json-parser.png)](http://travis-ci.org/udp/json-parser) - -_Want to serialize? Check out [json-builder](https://github.com/udp/json-builder)!_ - -Installing ----------- - -There is now a makefile which will produce a libjsonparser static and dynamic library. However, this -is _not_ required to build json-parser, and the source files (`json.c` and `json.h`) should be happy -in any build system you already have in place. - - -API ---- - - json_value * json_parse (const json_char * json, - size_t length); - - json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error); - - void json_value_free (json_value *); - -The `type` field of `json_value` is one of: - -* `json_object` (see `u.object.length`, `u.object.values[x].name`, `u.object.values[x].value`) -* `json_array` (see `u.array.length`, `u.array.values`) -* `json_integer` (see `u.integer`) -* `json_double` (see `u.dbl`) -* `json_string` (see `u.string.ptr`, `u.string.length`) -* `json_boolean` (see `u.boolean`) -* `json_null` - - -Compile-Time Options --------------------- - - -DJSON_TRACK_SOURCE - -Stores the source location (line and column number) inside each `json_value`. - -This is useful for application-level error reporting. - - -Runtime Options ---------------- - - settings |= json_enable_comments; - -Enables C-style `// line` and `/* block */` comments. - - size_t value_extra - -The amount of space (if any) to allocate at the end of each `json_value`, in -order to give the application space to add metadata. - - void * (* mem_alloc) (size_t, int zero, void * user_data); - void (* mem_free) (void *, void * user_data); - -Custom allocator routines. If NULL, the default `malloc` and `free` will be used. - -The `user_data` pointer will be forwarded from `json_settings` to allow application -context to be passed. - - -Changes in version 1.1.0 ------------------------- - -* UTF-8 byte order marks are now skipped if present - -* Allows cross-compilation by honoring --host if given (@wkz) - -* Maximum size for error buffer is now exposed in header (@LB--) - -* GCC warning for `static` after `const` fixed (@batrick) - -* Optional support for C-style line and block comments added (@Jin-W-FS) - -* `name_length` field added to object values - -* It is now possible to retrieve the source line/column number of a parsed `json_value` when `JSON_TRACK_SOURCE` is enabled - -* The application may now extend `json_value` using the `value_extra` setting - -* Un-ambiguate pow call in the case of C++ overloaded pow (@fcartegnie) - -* Fix null pointer de-reference when a non-existing array is closed and no root value is present - - diff --git a/ext/json-parser/json.c b/ext/json-parser/json.c deleted file mode 100644 index 166cdcb6..00000000 --- a/ext/json-parser/json.c +++ /dev/null @@ -1,1012 +0,0 @@ -/* vim: set et ts=3 sw=3 sts=3 ft=c: - * - * Copyright (C) 2012, 2013, 2014 James McLaughlin et al. All rights reserved. - * https://github.com/udp/json-parser - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "json.h" - -#ifdef _MSC_VER - #ifndef _CRT_SECURE_NO_WARNINGS - #define _CRT_SECURE_NO_WARNINGS - #endif - #pragma warning(disable:4996) -#endif - -const struct _json_value json_value_none; - -#include -#include -#include -#include - -typedef unsigned int json_uchar; - -static unsigned char hex_value (json_char c) -{ - if (isdigit(c)) - return c - '0'; - - switch (c) { - case 'a': case 'A': return 0x0A; - case 'b': case 'B': return 0x0B; - case 'c': case 'C': return 0x0C; - case 'd': case 'D': return 0x0D; - case 'e': case 'E': return 0x0E; - case 'f': case 'F': return 0x0F; - default: return 0xFF; - } -} - -typedef struct -{ - unsigned long used_memory; - - unsigned int uint_max; - unsigned long ulong_max; - - json_settings settings; - int first_pass; - - const json_char * ptr; - unsigned int cur_line, cur_col; - -} json_state; - -static void * default_alloc (size_t size, int zero, void * user_data) -{ - return zero ? calloc (1, size) : malloc (size); -} - -static void default_free (void * ptr, void * user_data) -{ - free (ptr); -} - -static void * json_alloc (json_state * state, unsigned long size, int zero) -{ - if ((state->ulong_max - state->used_memory) < size) - return 0; - - if (state->settings.max_memory - && (state->used_memory += size) > state->settings.max_memory) - { - return 0; - } - - return state->settings.mem_alloc (size, zero, state->settings.user_data); -} - -static int new_value (json_state * state, - json_value ** top, json_value ** root, json_value ** alloc, - json_type type) -{ - json_value * value; - int values_size; - - if (!state->first_pass) - { - value = *top = *alloc; - *alloc = (*alloc)->_reserved.next_alloc; - - if (!*root) - *root = value; - - switch (value->type) - { - case json_array: - - if (value->u.array.length == 0) - break; - - if (! (value->u.array.values = (json_value **) json_alloc - (state, value->u.array.length * sizeof (json_value *), 0)) ) - { - return 0; - } - - value->u.array.length = 0; - break; - - case json_object: - - if (value->u.object.length == 0) - break; - - values_size = sizeof (*value->u.object.values) * value->u.object.length; - - if (! (value->u.object.values = (json_object_entry *) json_alloc - (state, values_size + ((unsigned long) value->u.object.values), 0)) ) - { - return 0; - } - - value->_reserved.object_mem = (*(char **) &value->u.object.values) + values_size; - - value->u.object.length = 0; - break; - - case json_string: - - if (! (value->u.string.ptr = (json_char *) json_alloc - (state, (value->u.string.length + 1) * sizeof (json_char), 0)) ) - { - return 0; - } - - value->u.string.length = 0; - break; - - default: - break; - }; - - return 1; - } - - if (! (value = (json_value *) json_alloc - (state, sizeof (json_value) + state->settings.value_extra, 1))) - { - return 0; - } - - if (!*root) - *root = value; - - value->type = type; - value->parent = *top; - - #ifdef JSON_TRACK_SOURCE - value->line = state->cur_line; - value->col = state->cur_col; - #endif - - if (*alloc) - (*alloc)->_reserved.next_alloc = value; - - *alloc = *top = value; - - return 1; -} - -#define whitespace \ - case '\n': ++ state.cur_line; state.cur_col = 0; \ - case ' ': case '\t': case '\r' - -#define string_add(b) \ - do { if (!state.first_pass) string [string_length] = b; ++ string_length; } while (0); - -#define line_and_col \ - state.cur_line, state.cur_col - -static const long - flag_next = 1 << 0, - flag_reproc = 1 << 1, - flag_need_comma = 1 << 2, - flag_seek_value = 1 << 3, - flag_escaped = 1 << 4, - flag_string = 1 << 5, - flag_need_colon = 1 << 6, - flag_done = 1 << 7, - flag_num_negative = 1 << 8, - flag_num_zero = 1 << 9, - flag_num_e = 1 << 10, - flag_num_e_got_sign = 1 << 11, - flag_num_e_negative = 1 << 12, - flag_line_comment = 1 << 13, - flag_block_comment = 1 << 14; - -json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error_buf) -{ - json_char error [json_error_max]; - const json_char * end; - json_value * top, * root, * alloc = 0; - json_state state = { 0 }; - long flags; - long num_digits = 0, num_e = 0; - json_int_t num_fraction = 0; - - /* Skip UTF-8 BOM - */ - if (length >= 3 && ((unsigned char) json [0]) == 0xEF - && ((unsigned char) json [1]) == 0xBB - && ((unsigned char) json [2]) == 0xBF) - { - json += 3; - length -= 3; - } - - error[0] = '\0'; - end = (json + length); - - memcpy (&state.settings, settings, sizeof (json_settings)); - - if (!state.settings.mem_alloc) - state.settings.mem_alloc = default_alloc; - - if (!state.settings.mem_free) - state.settings.mem_free = default_free; - - memset (&state.uint_max, 0xFF, sizeof (state.uint_max)); - memset (&state.ulong_max, 0xFF, sizeof (state.ulong_max)); - - state.uint_max -= 8; /* limit of how much can be added before next check */ - state.ulong_max -= 8; - - for (state.first_pass = 1; state.first_pass >= 0; -- state.first_pass) - { - json_uchar uchar; - unsigned char uc_b1, uc_b2, uc_b3, uc_b4; - json_char * string = 0; - unsigned int string_length = 0; - - top = root = 0; - flags = flag_seek_value; - - state.cur_line = 1; - - for (state.ptr = json ;; ++ state.ptr) - { - json_char b = (state.ptr == end ? 0 : *state.ptr); - - if (flags & flag_string) - { - if (!b) - { sprintf (error, "Unexpected EOF in string (at %d:%d)", line_and_col); - goto e_failed; - } - - if (string_length > state.uint_max) - goto e_overflow; - - if (flags & flag_escaped) - { - flags &= ~ flag_escaped; - - switch (b) - { - case 'b': string_add ('\b'); break; - case 'f': string_add ('\f'); break; - case 'n': string_add ('\n'); break; - case 'r': string_add ('\r'); break; - case 't': string_add ('\t'); break; - case 'u': - - if (end - state.ptr < 4 || - (uc_b1 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b2 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b3 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b4 = hex_value (*++ state.ptr)) == 0xFF) - { - sprintf (error, "Invalid character value `%c` (at %d:%d)", b, line_and_col); - goto e_failed; - } - - uc_b1 = (uc_b1 << 4) | uc_b2; - uc_b2 = (uc_b3 << 4) | uc_b4; - uchar = (uc_b1 << 8) | uc_b2; - - if ((uchar & 0xF800) == 0xD800) { - json_uchar uchar2; - - if (end - state.ptr < 6 || (*++ state.ptr) != '\\' || (*++ state.ptr) != 'u' || - (uc_b1 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b2 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b3 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b4 = hex_value (*++ state.ptr)) == 0xFF) - { - sprintf (error, "Invalid character value `%c` (at %d:%d)", b, line_and_col); - goto e_failed; - } - - uc_b1 = (uc_b1 << 4) | uc_b2; - uc_b2 = (uc_b3 << 4) | uc_b4; - uchar2 = (uc_b1 << 8) | uc_b2; - - uchar = 0x010000 | ((uchar & 0x3FF) << 10) | (uchar2 & 0x3FF); - } - - if (sizeof (json_char) >= sizeof (json_uchar) || (uchar <= 0x7F)) - { - string_add ((json_char) uchar); - break; - } - - if (uchar <= 0x7FF) - { - if (state.first_pass) - string_length += 2; - else - { string [string_length ++] = 0xC0 | (uchar >> 6); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - } - - if (uchar <= 0xFFFF) { - if (state.first_pass) - string_length += 3; - else - { string [string_length ++] = 0xE0 | (uchar >> 12); - string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - } - - if (state.first_pass) - string_length += 4; - else - { string [string_length ++] = 0xF0 | (uchar >> 18); - string [string_length ++] = 0x80 | ((uchar >> 12) & 0x3F); - string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - - default: - string_add (b); - }; - - continue; - } - - if (b == '\\') - { - flags |= flag_escaped; - continue; - } - - if (b == '"') - { - if (!state.first_pass) - string [string_length] = 0; - - flags &= ~ flag_string; - string = 0; - - switch (top->type) - { - case json_string: - - top->u.string.length = string_length; - flags |= flag_next; - - break; - - case json_object: - - if (state.first_pass) - (*(json_char **) &top->u.object.values) += string_length + 1; - else - { - top->u.object.values [top->u.object.length].name - = (json_char *) top->_reserved.object_mem; - - top->u.object.values [top->u.object.length].name_length - = string_length; - - (*(json_char **) &top->_reserved.object_mem) += string_length + 1; - } - - flags |= flag_seek_value | flag_need_colon; - continue; - - default: - break; - }; - } - else - { - string_add (b); - continue; - } - } - - if (state.settings.settings & json_enable_comments) - { - if (flags & (flag_line_comment | flag_block_comment)) - { - if (flags & flag_line_comment) - { - if (b == '\r' || b == '\n' || !b) - { - flags &= ~ flag_line_comment; - -- state.ptr; /* so null can be reproc'd */ - } - - continue; - } - - if (flags & flag_block_comment) - { - if (!b) - { sprintf (error, "%d:%d: Unexpected EOF in block comment", line_and_col); - goto e_failed; - } - - if (b == '*' && state.ptr < (end - 1) && state.ptr [1] == '/') - { - flags &= ~ flag_block_comment; - ++ state.ptr; /* skip closing sequence */ - } - - continue; - } - } - else if (b == '/') - { - if (! (flags & (flag_seek_value | flag_done)) && top->type != json_object) - { sprintf (error, "%d:%d: Comment not allowed here", line_and_col); - goto e_failed; - } - - if (++ state.ptr == end) - { sprintf (error, "%d:%d: EOF unexpected", line_and_col); - goto e_failed; - } - - switch (b = *state.ptr) - { - case '/': - flags |= flag_line_comment; - continue; - - case '*': - flags |= flag_block_comment; - continue; - - default: - sprintf (error, "%d:%d: Unexpected `%c` in comment opening sequence", line_and_col, b); - goto e_failed; - }; - } - } - - if (flags & flag_done) - { - if (!b) - break; - - switch (b) - { - whitespace: - continue; - - default: - - sprintf (error, "%d:%d: Trailing garbage: `%c`", - state.cur_line, state.cur_col, b); - - goto e_failed; - }; - } - - if (flags & flag_seek_value) - { - switch (b) - { - whitespace: - continue; - - case ']': - - if (top && top->type == json_array) - flags = (flags & ~ (flag_need_comma | flag_seek_value)) | flag_next; - else - { sprintf (error, "%d:%d: Unexpected ]", line_and_col); - goto e_failed; - } - - break; - - default: - - if (flags & flag_need_comma) - { - if (b == ',') - { flags &= ~ flag_need_comma; - continue; - } - else - { - sprintf (error, "%d:%d: Expected , before %c", - state.cur_line, state.cur_col, b); - - goto e_failed; - } - } - - if (flags & flag_need_colon) - { - if (b == ':') - { flags &= ~ flag_need_colon; - continue; - } - else - { - sprintf (error, "%d:%d: Expected : before %c", - state.cur_line, state.cur_col, b); - - goto e_failed; - } - } - - flags &= ~ flag_seek_value; - - switch (b) - { - case '{': - - if (!new_value (&state, &top, &root, &alloc, json_object)) - goto e_alloc_failure; - - continue; - - case '[': - - if (!new_value (&state, &top, &root, &alloc, json_array)) - goto e_alloc_failure; - - flags |= flag_seek_value; - continue; - - case '"': - - if (!new_value (&state, &top, &root, &alloc, json_string)) - goto e_alloc_failure; - - flags |= flag_string; - - string = top->u.string.ptr; - string_length = 0; - - continue; - - case 't': - - if ((end - state.ptr) < 3 || *(++ state.ptr) != 'r' || - *(++ state.ptr) != 'u' || *(++ state.ptr) != 'e') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_boolean)) - goto e_alloc_failure; - - top->u.boolean = 1; - - flags |= flag_next; - break; - - case 'f': - - if ((end - state.ptr) < 4 || *(++ state.ptr) != 'a' || - *(++ state.ptr) != 'l' || *(++ state.ptr) != 's' || - *(++ state.ptr) != 'e') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_boolean)) - goto e_alloc_failure; - - flags |= flag_next; - break; - - case 'n': - - if ((end - state.ptr) < 3 || *(++ state.ptr) != 'u' || - *(++ state.ptr) != 'l' || *(++ state.ptr) != 'l') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_null)) - goto e_alloc_failure; - - flags |= flag_next; - break; - - default: - - if (isdigit (b) || b == '-') - { - if (!new_value (&state, &top, &root, &alloc, json_integer)) - goto e_alloc_failure; - - if (!state.first_pass) - { - while (isdigit (b) || b == '+' || b == '-' - || b == 'e' || b == 'E' || b == '.') - { - if ( (++ state.ptr) == end) - { - b = 0; - break; - } - - b = *state.ptr; - } - - flags |= flag_next | flag_reproc; - break; - } - - flags &= ~ (flag_num_negative | flag_num_e | - flag_num_e_got_sign | flag_num_e_negative | - flag_num_zero); - - num_digits = 0; - num_fraction = 0; - num_e = 0; - - if (b != '-') - { - flags |= flag_reproc; - break; - } - - flags |= flag_num_negative; - continue; - } - else - { sprintf (error, "%d:%d: Unexpected %c when seeking value", line_and_col, b); - goto e_failed; - } - }; - }; - } - else - { - switch (top->type) - { - case json_object: - - switch (b) - { - whitespace: - continue; - - case '"': - - if (flags & flag_need_comma) - { sprintf (error, "%d:%d: Expected , before \"", line_and_col); - goto e_failed; - } - - flags |= flag_string; - - string = (json_char *) top->_reserved.object_mem; - string_length = 0; - - break; - - case '}': - - flags = (flags & ~ flag_need_comma) | flag_next; - break; - - case ',': - - if (flags & flag_need_comma) - { - flags &= ~ flag_need_comma; - break; - } - - default: - sprintf (error, "%d:%d: Unexpected `%c` in object", line_and_col, b); - goto e_failed; - }; - - break; - - case json_integer: - case json_double: - - if (isdigit (b)) - { - ++ num_digits; - - if (top->type == json_integer || flags & flag_num_e) - { - if (! (flags & flag_num_e)) - { - if (flags & flag_num_zero) - { sprintf (error, "%d:%d: Unexpected `0` before `%c`", line_and_col, b); - goto e_failed; - } - - if (num_digits == 1 && b == '0') - flags |= flag_num_zero; - } - else - { - flags |= flag_num_e_got_sign; - num_e = (num_e * 10) + (b - '0'); - continue; - } - - top->u.integer = (top->u.integer * 10) + (b - '0'); - continue; - } - - num_fraction = (num_fraction * 10) + (b - '0'); - continue; - } - - if (b == '+' || b == '-') - { - if ( (flags & flag_num_e) && !(flags & flag_num_e_got_sign)) - { - flags |= flag_num_e_got_sign; - - if (b == '-') - flags |= flag_num_e_negative; - - continue; - } - } - else if (b == '.' && top->type == json_integer) - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit before `.`", line_and_col); - goto e_failed; - } - - top->type = json_double; - top->u.dbl = (double) top->u.integer; - - num_digits = 0; - continue; - } - - if (! (flags & flag_num_e)) - { - if (top->type == json_double) - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit after `.`", line_and_col); - goto e_failed; - } - - top->u.dbl += ((double) num_fraction) / (pow (10.0, (double) num_digits)); - } - - if (b == 'e' || b == 'E') - { - flags |= flag_num_e; - - if (top->type == json_integer) - { - top->type = json_double; - top->u.dbl = (double) top->u.integer; - } - - num_digits = 0; - flags &= ~ flag_num_zero; - - continue; - } - } - else - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit after `e`", line_and_col); - goto e_failed; - } - - top->u.dbl *= pow (10.0, (double) - (flags & flag_num_e_negative ? - num_e : num_e)); - } - - if (flags & flag_num_negative) - { - if (top->type == json_integer) - top->u.integer = - top->u.integer; - else - top->u.dbl = - top->u.dbl; - } - - flags |= flag_next | flag_reproc; - break; - - default: - break; - }; - } - - if (flags & flag_reproc) - { - flags &= ~ flag_reproc; - -- state.ptr; - } - - if (flags & flag_next) - { - flags = (flags & ~ flag_next) | flag_need_comma; - - if (!top->parent) - { - /* root value done */ - - flags |= flag_done; - continue; - } - - if (top->parent->type == json_array) - flags |= flag_seek_value; - - if (!state.first_pass) - { - json_value * parent = top->parent; - - switch (parent->type) - { - case json_object: - - parent->u.object.values - [parent->u.object.length].value = top; - - break; - - case json_array: - - parent->u.array.values - [parent->u.array.length] = top; - - break; - - default: - break; - }; - } - - if ( (++ top->parent->u.array.length) > state.uint_max) - goto e_overflow; - - top = top->parent; - - continue; - } - } - - alloc = root; - } - - return root; - -e_unknown_value: - - sprintf (error, "%d:%d: Unknown value", line_and_col); - goto e_failed; - -e_alloc_failure: - - strcpy (error, "Memory allocation failure"); - goto e_failed; - -e_overflow: - - sprintf (error, "%d:%d: Too long (caught overflow)", line_and_col); - goto e_failed; - -e_failed: - - if (error_buf) - { - if (*error) - strcpy (error_buf, error); - else - strcpy (error_buf, "Unknown error"); - } - - if (state.first_pass) - alloc = root; - - while (alloc) - { - top = alloc->_reserved.next_alloc; - state.settings.mem_free (alloc, state.settings.user_data); - alloc = top; - } - - if (!state.first_pass) - json_value_free_ex (&state.settings, root); - - return 0; -} - -json_value * json_parse (const json_char * json, size_t length) -{ - json_settings settings = { 0 }; - return json_parse_ex (&settings, json, length, 0); -} - -void json_value_free_ex (json_settings * settings, json_value * value) -{ - json_value * cur_value; - - if (!value) - return; - - value->parent = 0; - - while (value) - { - switch (value->type) - { - case json_array: - - if (!value->u.array.length) - { - settings->mem_free (value->u.array.values, settings->user_data); - break; - } - - value = value->u.array.values [-- value->u.array.length]; - continue; - - case json_object: - - if (!value->u.object.length) - { - settings->mem_free (value->u.object.values, settings->user_data); - break; - } - - value = value->u.object.values [-- value->u.object.length].value; - continue; - - case json_string: - - settings->mem_free (value->u.string.ptr, settings->user_data); - break; - - default: - break; - }; - - cur_value = value; - value = value->parent; - settings->mem_free (cur_value, settings->user_data); - } -} - -void json_value_free (json_value * value) -{ - json_settings settings = { 0 }; - settings.mem_free = default_free; - json_value_free_ex (&settings, value); -} - diff --git a/ext/json-parser/json.h b/ext/json-parser/json.h deleted file mode 100644 index f6549ec4..00000000 --- a/ext/json-parser/json.h +++ /dev/null @@ -1,283 +0,0 @@ - -/* vim: set et ts=3 sw=3 sts=3 ft=c: - * - * Copyright (C) 2012, 2013, 2014 James McLaughlin et al. All rights reserved. - * https://github.com/udp/json-parser - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef _JSON_H -#define _JSON_H - -#ifndef json_char - #define json_char char -#endif - -#ifndef json_int_t - #ifndef _MSC_VER - #include - #define json_int_t int64_t - #else - #define json_int_t __int64 - #endif -#endif - -#include - -#ifdef __cplusplus - - #include - - extern "C" - { - -#endif - -typedef struct -{ - unsigned long max_memory; - int settings; - - /* Custom allocator support (leave null to use malloc/free) - */ - - void * (* mem_alloc) (size_t, int zero, void * user_data); - void (* mem_free) (void *, void * user_data); - - void * user_data; /* will be passed to mem_alloc and mem_free */ - - size_t value_extra; /* how much extra space to allocate for values? */ - -} json_settings; - -#define json_enable_comments 0x01 - -typedef enum -{ - json_none, - json_object, - json_array, - json_integer, - json_double, - json_string, - json_boolean, - json_null - -} json_type; - -extern const struct _json_value json_value_none; - -typedef struct _json_object_entry -{ - json_char * name; - unsigned int name_length; - - struct _json_value * value; - -} json_object_entry; - -typedef struct _json_value -{ - struct _json_value * parent; - - json_type type; - - union - { - int boolean; - json_int_t integer; - double dbl; - - struct - { - unsigned int length; - json_char * ptr; /* null terminated */ - - } string; - - struct - { - unsigned int length; - - json_object_entry * values; - - #if defined(__cplusplus) && __cplusplus >= 201103L - decltype(values) begin () const - { return values; - } - decltype(values) end () const - { return values + length; - } - #endif - - } object; - - struct - { - unsigned int length; - struct _json_value ** values; - - #if defined(__cplusplus) && __cplusplus >= 201103L - decltype(values) begin () const - { return values; - } - decltype(values) end () const - { return values + length; - } - #endif - - } array; - - } u; - - union - { - struct _json_value * next_alloc; - void * object_mem; - - } _reserved; - - #ifdef JSON_TRACK_SOURCE - - /* Location of the value in the source JSON - */ - unsigned int line, col; - - #endif - - - /* Some C++ operator sugar */ - - #ifdef __cplusplus - - public: - - inline _json_value () - { memset (this, 0, sizeof (_json_value)); - } - - inline const struct _json_value &operator [] (int index) const - { - if (type != json_array || index < 0 - || ((unsigned int) index) >= u.array.length) - { - return json_value_none; - } - - return *u.array.values [index]; - } - - inline const struct _json_value &operator [] (const char * index) const - { - if (type != json_object) - return json_value_none; - - for (unsigned int i = 0; i < u.object.length; ++ i) - if (!strcmp (u.object.values [i].name, index)) - return *u.object.values [i].value; - - return json_value_none; - } - - inline operator const char * () const - { - switch (type) - { - case json_string: - return u.string.ptr; - - default: - return ""; - }; - } - - inline operator json_int_t () const - { - switch (type) - { - case json_integer: - return u.integer; - - case json_double: - return (json_int_t) u.dbl; - - default: - return 0; - }; - } - - inline operator bool () const - { - if (type != json_boolean) - return false; - - return u.boolean != 0; - } - - inline operator double () const - { - switch (type) - { - case json_integer: - return (double) u.integer; - - case json_double: - return u.dbl; - - default: - return 0; - }; - } - - #endif - -} json_value; - -json_value * json_parse (const json_char * json, - size_t length); - -#define json_error_max 128 -json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error); - -void json_value_free (json_value *); - - -/* Not usually necessary, unless you used a custom mem_alloc and now want to - * use a custom mem_free. - */ -void json_value_free_ex (json_settings * settings, - json_value *); - - -#ifdef __cplusplus - } /* extern "C" */ -#endif - -#endif - - diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index aa7ecc2c..5864e346 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -468,6 +468,11 @@ enum ZT_VirtualNetworkType ZT_NETWORK_TYPE_PUBLIC = 1 }; +/* + - TEE : should use a field to indicate how many bytes of each packet max are TEE'd + - Controller : web hooks for auth, optional required re-auth? or auth for a period of time? auto-expiring auth? +*/ + /** * The type of a virtual network rules table entry * @@ -721,6 +726,15 @@ typedef struct uint32_t id; uint32_t value; } tag; + + /** + * Destinations for TEE and REDIRECT + */ + struct { + uint64_t address; + uint32_t flags; + uint16_t length; + } fwd; } v; } ZT_VirtualNetworkRule; diff --git a/make-freebsd.mk b/make-freebsd.mk index e7bd9fd2..cb9a2e6d 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -6,7 +6,7 @@ DEFS= LIBS= include objects.mk -OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/json-parser/json.o ext/http-parser/http_parser.o +OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # "make official" is a shortcut for this ifeq ($(ZT_OFFICIAL_RELEASE),1) diff --git a/make-linux.mk b/make-linux.mk index fe9ecad8..1fc27bde 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -51,12 +51,6 @@ else LDLIBS+=-lhttp_parser DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER endif -ifeq ($(wildcard /usr/include/json-parser/json.h),) - OBJS+=ext/json-parser/json.o -else - LDLIBS+=-ljsonparser - DEFS+=-DZT_USE_SYSTEM_JSON_PARSER -endif ifeq ($(ZT_USE_MINIUPNPC),1) OBJS+=osdep/PortMapper.o diff --git a/make-mac.mk b/make-mac.mk index f00f8d6b..ee90ae4c 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -11,7 +11,7 @@ LIBS= ARCH_FLAGS=-arch x86_64 include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/json-parser/json.o ext/http-parser/http_parser.o +OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # Disable codesign since open source users will not have ZeroTier's certs CODESIGN=echo diff --git a/node/Capability.hpp b/node/Capability.hpp index b0620891..0b352725 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -181,6 +181,11 @@ public: break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: + b.append((uint8_t)14); + b.append((uint64_t)rules[i].v.fwd.address); + b.append((uint32_t)rules[i].v.fwd.flags); + b.append((uint16_t)rules[i].v.fwd.length); + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: b.append((uint8_t)5); @@ -266,6 +271,10 @@ public: break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: + rules[ruleCount].v.fwd.address = b.template at(p); + rules[ruleCount].v.fwd.flags = b.template at(p + 8); + rules[ruleCount].v.fwd.length = b.template at(p + 12); + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: rules[ruleCount].v.zt = Address(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); diff --git a/node/Network.cpp b/node/Network.cpp index 1319df4e..e12dd027 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -154,13 +154,13 @@ static int _doZtFilter( break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { - Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); + Packet outp(Address(rules[rn].v.fwd.address),RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(nconf.networkId); - outp.append((uint8_t)((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02)); + outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); - outp.append(frameData,frameLen); + outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); outp.compress(); RR->sw->send(outp,true); diff --git a/node/Packet.hpp b/node/Packet.hpp index 0a5d3fec..570bace9 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -657,6 +657,7 @@ public: * 0x01 - Certificate of network membership attached (DEPRECATED) * 0x02 - Packet is a TEE'd packet * 0x04 - Packet is a REDIRECT'ed packet + * 0x08 - TEE/REDIRECT'ed packet is on inbound side of connection * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we diff --git a/one.cpp b/one.cpp index 9f7a0a29..f3442933 100644 --- a/one.cpp +++ b/one.cpp @@ -48,16 +48,12 @@ #include #include +#include +#include #include "version.h" #include "include/ZeroTierOne.h" -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "ext/json-parser/json.h" -#endif - #include "node/Identity.hpp" #include "node/CertificateOfMembership.hpp" #include "node/Utils.hpp" @@ -68,6 +64,8 @@ #include "service/OneService.hpp" +#include "ext/json/json.hpp" + #define ZT_PID_PATH "zerotier-one.pid" using namespace ZeroTier; @@ -283,221 +281,135 @@ static int cli(int argc,char **argv) return 1; } } else if ((command == "info")||(command == "status")) { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/status", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/status",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - bool good = false; - if (j) { - if (j->type == json_object) { - const char *address = (const char *)0; - bool online = false; - const char *version = (const char *)0; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(j->u.object.values[k].name,"address"))&&(j->u.object.values[k].value->type == json_string)) - address = j->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(j->u.object.values[k].name,"version"))&&(j->u.object.values[k].value->type == json_string)) - version = j->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(j->u.object.values[k].name,"online"))&&(j->u.object.values[k].value->type == json_boolean)) - online = (j->u.object.values[k].value->u.boolean != 0); - } - if ((address)&&(version)) { - printf("200 info %s %s %s" ZT_EOL_S,address,(online ? "ONLINE" : "OFFLINE"),version); - good = true; - } - } - json_value_free(j); - } - if (good) { - return 0; - } else { - printf("%u %s invalid JSON response" ZT_EOL_S,scode,command.c_str()); - return 1; - } + if (j.is_object()) + out << "200 info " << j["address"].get() << " " << j["version"].get() << " " << ((j["tcpFallbackActive"]) ? "TUNNELED" : ((j["online"]) ? "ONLINE" : "OFFLINE")) << ZT_EOL_S; } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } } else if (command == "listpeers") { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/peer", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/peer",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - printf("200 listpeers " ZT_EOL_S); - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - if (j) { - if (j->type == json_array) { - for(unsigned int p=0;pu.array.length;++p) { - json_value *jp = j->u.array.values[p]; - if (jp->type == json_object) { - const char *address = (const char *)0; - std::string paths; - int64_t latency = 0; - int64_t versionMajor = -1,versionMinor = -1,versionRev = -1; - const char *role = (const char *)0; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(jp->u.object.values[k].name,"address"))&&(jp->u.object.values[k].value->type == json_string)) - address = jp->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jp->u.object.values[k].name,"versionMajor"))&&(jp->u.object.values[k].value->type == json_integer)) - versionMajor = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"versionMinor"))&&(jp->u.object.values[k].value->type == json_integer)) - versionMinor = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"versionRev"))&&(jp->u.object.values[k].value->type == json_integer)) - versionRev = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"role"))&&(jp->u.object.values[k].value->type == json_string)) - role = jp->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jp->u.object.values[k].name,"latency"))&&(jp->u.object.values[k].value->type == json_integer)) - latency = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"paths"))&&(jp->u.object.values[k].value->type == json_array)) { - for(unsigned int pp=0;ppu.object.values[k].value->u.array.length;++pp) { - json_value *jpath = jp->u.object.values[k].value->u.array.values[pp]; - if (jpath->type == json_object) { - const char *paddr = (const char *)0; - int64_t lastSend = 0; - int64_t lastReceive = 0; - bool preferred = false; - bool active = false; - for(unsigned int kk=0;kku.object.length;++kk) { - if ((!strcmp(jpath->u.object.values[kk].name,"address"))&&(jpath->u.object.values[kk].value->type == json_string)) - paddr = jpath->u.object.values[kk].value->u.string.ptr; - else if ((!strcmp(jpath->u.object.values[kk].name,"lastSend"))&&(jpath->u.object.values[kk].value->type == json_integer)) - lastSend = jpath->u.object.values[kk].value->u.integer; - else if ((!strcmp(jpath->u.object.values[kk].name,"lastReceive"))&&(jpath->u.object.values[kk].value->type == json_integer)) - lastReceive = jpath->u.object.values[kk].value->u.integer; - else if ((!strcmp(jpath->u.object.values[kk].name,"preferred"))&&(jpath->u.object.values[kk].value->type == json_boolean)) - preferred = (jpath->u.object.values[kk].value->u.boolean != 0); - else if ((!strcmp(jpath->u.object.values[kk].name,"active"))&&(jpath->u.object.values[kk].value->type == json_boolean)) - active = (jpath->u.object.values[kk].value->u.boolean != 0); - } - if ((paddr)&&(active)) { - int64_t now = (int64_t)OSUtils::now(); - if (lastSend > 0) - lastSend = now - lastSend; - if (lastReceive > 0) - lastReceive = now - lastReceive; - char pathtmp[256]; - Utils::snprintf(pathtmp,sizeof(pathtmp),"%s;%lld;%lld;%s", - paddr, - lastSend, - lastReceive, - (preferred ? "preferred" : "active")); - if (paths.length()) - paths.push_back(','); - paths.append(pathtmp); - } - } - } - } - } - if ((address)&&(role)) { - char verstr[64]; - if ((versionMajor >= 0)&&(versionMinor >= 0)&&(versionRev >= 0)) - Utils::snprintf(verstr,sizeof(verstr),"%lld.%lld.%lld",versionMajor,versionMinor,versionRev); - else { - verstr[0] = '-'; - verstr[1] = (char)0; - } - printf("200 listpeers %s %s %lld %s %s" ZT_EOL_S,address,(paths.length()) ? paths.c_str() : "-",(long long)latency,verstr,role); + out << "200 listpeers " << ZT_EOL_S; + if (j.is_array()) { + for(unsigned long k=0;k= 0) { + Utils::snprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + } else { + ver[0] = '-'; + ver[1] = (char)0; + } + out << "200 listpeers " << p["address"].get() << " " << bestPath << " " << p["latency"] << " " << ver << " " << p["role"].get() << ZT_EOL_S; } - json_value_free(j); } - return 0; } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } } else if (command == "listnetworks") { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/network", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/network",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - printf("200 listnetworks " ZT_EOL_S); - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - if (j) { - if (j->type == json_array) { - for(unsigned int p=0;pu.array.length;++p) { - json_value *jn = j->u.array.values[p]; - if (jn->type == json_object) { - const char *nwid = (const char *)0; - const char *name = ""; - const char *mac = (const char *)0; - const char *status = (const char *)0; - const char *type = (const char *)0; - const char *portDeviceName = ""; - std::string ips; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(jn->u.object.values[k].name,"nwid"))&&(jn->u.object.values[k].value->type == json_string)) - nwid = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"name"))&&(jn->u.object.values[k].value->type == json_string)) - name = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"mac"))&&(jn->u.object.values[k].value->type == json_string)) - mac = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"status"))&&(jn->u.object.values[k].value->type == json_string)) - status = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"type"))&&(jn->u.object.values[k].value->type == json_string)) - type = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"portDeviceName"))&&(jn->u.object.values[k].value->type == json_string)) - portDeviceName = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"assignedAddresses"))&&(jn->u.object.values[k].value->type == json_array)) { - for(unsigned int a=0;au.object.values[k].value->u.array.length;++a) { - json_value *aa = jn->u.object.values[k].value->u.array.values[a]; - if (aa->type == json_string) { - if (ips.length()) - ips.push_back(','); - ips.append(aa->u.string.ptr); - } - } + out << "200 listnetworks " << ZT_EOL_S; + if (j.is_array()) { + for(unsigned long i=0;i 0) aa.push_back(','); + aa.append(addr); } } - if ((nwid)&&(mac)&&(status)&&(type)) { - printf("200 listnetworks %s %s %s %s %s %s %s" ZT_EOL_S, - nwid, - (((name)&&(name[0])) ? name : "-"), - mac, - status, - type, - (((portDeviceName)&&(portDeviceName[0])) ? portDeviceName : "-"), - ((ips.length() > 0) ? ips.c_str() : "-")); - } } + if (aa.length() == 0) aa = "-"; + out << "200 listnetworks " << n["nwid"].get() << " " << n["name"].get() << " " << n["mac"].get() << " " << n["status"].get() << " " << n["type"].get() << " " << n["portDeviceName"].get() << " " << aa << ZT_EOL_S; } } - json_value_free(j); } } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 7aa757a9..5ed6b8b7 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -28,11 +28,7 @@ #include "../ext/http-parser/http_parser.h" #endif -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "../ext/json-parser/json.h" -#endif +#include "../ext/json/json.hpp" #include "../controller/EmbeddedNetworkController.hpp" @@ -519,23 +515,18 @@ unsigned int ControlPlane::handleRequest( OneService::NetworkSettings localSettings; _svc->getNetworkSettings(nws->networks[i].nwid,localSettings); - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - if (!strcmp(j->u.object.values[k].name,"allowManaged")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowManaged = (j->u.object.values[k].value->u.boolean != 0); - } else if (!strcmp(j->u.object.values[k].name,"allowGlobal")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowGlobal = (j->u.object.values[k].value->u.boolean != 0); - } else if (!strcmp(j->u.object.values[k].name,"allowDefault")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowDefault = (j->u.object.values[k].value->u.boolean != 0); - } - } + try { + nlohmann::json j(nlohmann::json::parse(body)); + if (j.is_object()) { + auto allowManaged = j["allowManaged"]; + if (allowManaged.is_boolean()) localSettings.allowManaged = (bool)allowManaged; + auto allowGlobal = j["allowGlobal"]; + if (allowGlobal.is_boolean()) localSettings.allowGlobal = (bool)allowGlobal; + auto allowDefault = j["allowDefault"]; + if (allowDefault.is_boolean()) localSettings.allowDefault = (bool)allowDefault; } - json_value_free(j); + } catch ( ... ) { + // discard invalid JSON } _svc->setNetworkSettings(nws->networks[i].nwid,localSettings); -- cgit v1.2.3 From ccea3d04d63e39a020e140b348aa87e272747c7e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 14:28:16 -0700 Subject: Push NETWORK_CONFIG_REFRESH on POSTs to /member/... in controller. --- controller/EmbeddedNetworkController.cpp | 2 ++ include/ZeroTierOne.h | 21 +++++++++++++++++++++ node/Network.cpp | 7 +++++++ node/Network.hpp | 5 +++++ node/Node.cpp | 32 ++++++++++++++++++++++++++++++++ node/Node.hpp | 1 + 6 files changed, 68 insertions(+) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0a4a17f8..9cf43b79 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1081,6 +1081,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); + _node->pushNetworkRefresh(address,nwid,(const uint64_t *)0,(const uint64_t *)0,0); + // Add non-persisted fields member["clock"] = now; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 5864e346..906edef2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1853,6 +1853,27 @@ enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,v */ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test); +/** + * Push a network refresh + * + * This is used by network controller implementations to send a + * NETWORK_CONFIG_REFRESH message to tell a node to refresh its + * config and to optionally push one or more credential timestamp + * blacklist thresholds for members of the network. + * + * Code outside a controller implementation will have no use for + * this as these messages are ignored if they do not come from a + * controller. + * + * @param node Node instance + * @param dest ZeroTier address of destination to which to send NETWORK_CONFIG_REFRESH + * @param nwid Network ID + * @param blacklistAddresses Array of ZeroTier addresses of network members to set timestamp blacklists for + * @param blacklistBeforeTimestamps Timestamps before which to blacklist credentials for each corresponding address in blacklistAddresses[] + * @param blacklistCount Size of blacklistAddresses[] and blacklistBeforeTimestamps[] + */ +void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); + /** * Initialize cluster operation * diff --git a/node/Network.cpp b/node/Network.cpp index e12dd027..97341ee2 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -402,6 +402,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _portInitialized(false), _inboundConfigPacketId(0), _lastConfigUpdate(0), + _lastRequestedConfiguration(0), _destroyed(false), _netconfFailure(NETCONF_FAILURE_NONE), _portError(0) @@ -691,6 +692,12 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d void Network::requestConfiguration() { + // Sanity limit: do not request more often than once per second + const uint64_t now = RR->node->now(); + if ((now - _lastRequestedConfiguration) < 1000ULL) + return; + _lastRequestedConfiguration = RR->node->now(); + Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); diff --git a/node/Network.hpp b/node/Network.hpp index 37154dc7..382b16aa 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -249,6 +249,10 @@ public: /** * Causes this network to request an updated configuration from its master node now + * + * There is a circuit breaker here to prevent this from being done more often + * than once per second. This is to prevent things like NETWORK_CONFIG_REFRESH + * from causing multiple requests. */ void requestConfiguration(); @@ -442,6 +446,7 @@ private: NetworkConfig _config; volatile uint64_t _lastConfigUpdate; + volatile uint64_t _lastRequestedConfiguration; volatile bool _destroyed; diff --git a/node/Node.cpp b/node/Node.cpp index 4da79347..ff564eee 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -548,6 +548,31 @@ void Node::circuitTestEnd(ZT_CircuitTest *test) } } +void Node::pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) +{ + Packet outp(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); + outp.append(nwid); + outp.addSize(2); + unsigned int c = 0; + for(unsigned int i=0;i= ZT_PROTO_MAX_PACKET_LENGTH) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); + RR->sw->send(outp,true); + outp = Packet(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); + outp.append(nwid); + outp.addSize(2); + c = 0; + } + Address(blacklistAddresses[i]).appendTo(outp); + outp.append(blacklistBeforeTimestamps[i]); + ++c; + } + if (c > 0) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); + RR->sw->send(outp,true); + } +} + ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -935,6 +960,13 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test) } catch ( ... ) {} } +void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) +{ + try { + reinterpret_cast(node)->pushNetworkRefresh(dest,nwid,blacklistAddresses,blacklistBeforeTimestamps,blacklistCount); + } catch ( ... ) {} +} + enum ZT_ResultCode ZT_Node_clusterInit( ZT_Node *node, unsigned int myId, diff --git a/node/Node.hpp b/node/Node.hpp index 98c4fd7c..3c0a5e92 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -102,6 +102,7 @@ public: void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); + void pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); ZT_ResultCode clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, -- cgit v1.2.3 From 2cdda38dc484246d169082e9bae4805cd5b08e6b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 15:26:18 -0700 Subject: It basically works... at least on current controllers. --- include/ZeroTierOne.h | 10 +++++++++ node/Membership.cpp | 7 +------ node/Network.cpp | 46 +++++++++++++++++++++++++---------------- node/Network.hpp | 2 ++ node/OutboundMulticast.cpp | 2 +- node/Switch.cpp | 51 ++++++++++++++++++++++++++-------------------- one.cpp | 4 +++- 7 files changed, 74 insertions(+), 48 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 906edef2..736280a7 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -169,6 +169,16 @@ extern "C" { */ #define ZT_RULE_PACKET_CHARACTERISTICS_INBOUND 0x8000000000000000ULL +/** + * Packet characteristics flag: multicast or broadcast destination MAC + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_MULTICAST 0x4000000000000000ULL + +/** + * Packet characteristics flag: broadcast destination MAC + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_BROADCAST 0x2000000000000000ULL + /** * Packet characteristics flag: TCP left-most reserved bit */ diff --git a/node/Membership.cpp b/node/Membership.cpp index 969032ff..59058a3e 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -81,8 +81,7 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) { - if (com.issuedTo() != RR->identity.address()) - return -1; + TRACE("addCredential(COM) for %.16llx signed by %s issued to %s",com.networkId(),com.signedBy().toString().c_str(),com.issuedTo().toString().c_str()); if (_com == com) return 0; const int vr = com.verify(RR); @@ -93,8 +92,6 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) { - if (tag.issuedTo() != RR->identity.address()) - return -1; TState *t = _tags.get(tag.id()); if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) return 0; @@ -112,8 +109,6 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap) { - if (cap.issuedTo() != RR->identity.address()) - return -1; std::map::iterator c(_caps.find(cap.id())); if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) return 0; diff --git a/node/Network.cpp b/node/Network.cpp index 97341ee2..ecbacd93 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -102,6 +102,7 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig // 0 == no match, -1 == match/drop, 1 == match/accept static int _doZtFilter( const RuntimeEnvironment *RR, + const bool noRedirect, const NetworkConfig &nconf, const bool inbound, const Address &ztSource, @@ -154,15 +155,17 @@ static int _doZtFilter( break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { - Packet outp(Address(rules[rn].v.fwd.address),RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(nconf.networkId); - outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); - outp.compress(); - RR->sw->send(outp,true); + if (!noRedirect) { + Packet outp(Address(rules[rn].v.fwd.address),RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(nconf.networkId); + outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); + outp.compress(); + RR->sw->send(outp,true); + } if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { return -1; // match, drop packet (we redirected it) @@ -318,6 +321,8 @@ static int _doZtFilter( break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { uint64_t cf = (inbound) ? ZT_RULE_PACKET_CHARACTERISTICS_INBOUND : 0ULL; + if (macDest.isMulticast()) cf |= ZT_RULE_PACKET_CHARACTERISTICS_MULTICAST; + if (macDest.isBroadcast()) cf |= ZT_RULE_PACKET_CHARACTERISTICS_BROADCAST; if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)&&(frameData[9] == 0x06)) { const unsigned int headerLen = 4 * (frameData[0] & 0xf); cf |= (uint64_t)frameData[headerLen + 13]; @@ -456,6 +461,7 @@ Network::~Network() } bool Network::filterOutgoingPacket( + const bool noRedirect, const Address &ztSource, const Address &ztDest, const MAC &macSource, @@ -475,21 +481,23 @@ bool Network::filterOutgoingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); + if (ztDest) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); return true; } for(unsigned int c=0;c<_config.capabilityCount;++c) { relevantLocalTagCount = 0; - switch (_doZtFilter(RR,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); + if (ztDest) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); return true; } } @@ -517,7 +525,7 @@ bool Network::filterIncomingPacket( Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -528,7 +536,7 @@ bool Network::filterIncomingPacket( const Capability *c; while ((c = mci.next(_config))) { relevantLocalTagCount = 0; - switch(_doZtFilter(RR,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { case -1: return false; case 1: @@ -698,6 +706,8 @@ void Network::requestConfiguration() return; _lastRequestedConfiguration = RR->node->now(); + const Address ctrl(controller()); + Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); @@ -710,7 +720,7 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); - if (controller() == RR->identity.address()) { + if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { NetworkConfig nconf; switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,nconf)) { @@ -732,9 +742,9 @@ void Network::requestConfiguration() } } - TRACE("requesting netconf for network %.16llx from controller %s",(unsigned long long)_id,controller().toString().c_str()); + TRACE("requesting netconf for network %.16llx from controller %s",(unsigned long long)_id,ctrl.toString().c_str()); - Packet outp(controller(),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REQUEST); + Packet outp(ctrl,RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REQUEST); outp.append((uint64_t)_id); const unsigned int rmdSize = rmd.sizeBytes(); outp.append((uint16_t)rmdSize); diff --git a/node/Network.hpp b/node/Network.hpp index 382b16aa..c5e7d570 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -86,6 +86,7 @@ public: * certain actions may be taken such as pushing credentials to ztDest and * sending a copy of the packet to a TEE or REDIRECT target. * + * @param noRedirect If true, do not TEE or REDIRECT -- this is set for secondary filtrations done in multicast and bridge send paths * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -97,6 +98,7 @@ public: * @return True if packet should be sent to destination peer */ bool filterOutgoingPacket( + const bool noRedirect, const Address &ztSource, const Address &ztDest, const MAC &macSource, diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index c9952927..6b583e7c 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -86,7 +86,7 @@ void OutboundMulticast::init( void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) { const SharedPtr nw(RR->node->network(_nwid)); - if ((nw)&&(nw->filterOutgoingPacket(RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr); diff --git a/node/Switch.cpp b/node/Switch.cpp index 37daff27..f6e4d1ab 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -437,7 +437,11 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),len); - if (!network->filterOutgoingPacket(RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + // We filter with a NULL destination ZeroTier address first. Filtrations + // for each ZT destination are also done in OutboundMulticast, but these + // set noRedirect to true. This prevents multiple TEEs and REDIRECTs for + // multicast packets. + if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } @@ -452,17 +456,13 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c etherType, data, len); - - return; - } - - if (to[0] == MAC::firstOctetForNetwork(network->id())) { + } else if (to[0] == MAC::firstOctetForNetwork(network->id())) { // Destination is another ZeroTier peer on the same network Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this SharedPtr toPeer(RR->topology->getPeer(toZT)); - if (!network->filterOutgoingPacket(RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } @@ -487,13 +487,17 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } //TRACE("%.16llx: UNICAST: %s -> %s etherType==%s(%.4x) vlanId==%u len==%u fromBridged==%d includeCom==%d",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),etherType,vlanId,len,(int)fromBridged,(int)includeCom); - - return; - } - - { + } else { // Destination is bridged behind a remote peer + // We filter with a NULL destination ZeroTier address first. Filtrations + // for each ZT destination are also done below. This is the same rationale + // and design as for multicast. + if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + return; + } + Address bridges[ZT_MAX_BRIDGE_SPAM]; unsigned int numBridges = 0; @@ -527,16 +531,19 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } for(unsigned int b=0;b bridgePeer(RR->topology->getPeer(bridges[b])); - Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(network->id()); - outp.append((uint8_t)0x00); - to.appendTo(outp); - from.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(data,len); - outp.compress(); - send(outp,true); + if (network->filterOutgoingPacket(true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId)) { + Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(network->id()); + outp.append((uint8_t)0x00); + to.appendTo(outp); + from.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(data,len); + outp.compress(); + send(outp,true); + } else { + TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + } } } } diff --git a/one.cpp b/one.cpp index f3442933..3cb6b775 100644 --- a/one.cpp +++ b/one.cpp @@ -403,7 +403,9 @@ static int cli(int argc,char **argv) } } if (aa.length() == 0) aa = "-"; - out << "200 listnetworks " << n["nwid"].get() << " " << n["name"].get() << " " << n["mac"].get() << " " << n["status"].get() << " " << n["type"].get() << " " << n["portDeviceName"].get() << " " << aa << ZT_EOL_S; + std::string name = n["name"]; + if (name.length() == 0) name = "-"; + out << "200 listnetworks " << n["nwid"].get() << " " << name << " " << n["mac"].get() << " " << n["status"].get() << " " << n["type"].get() << " " << n["portDeviceName"].get() << " " << aa << ZT_EOL_S; } } } -- cgit v1.2.3 From 63e8ad4cc3b91cf0de89012da1ef78deee1a566a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 15:45:37 -0700 Subject: TRACE stuff. --- node/Membership.cpp | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index 59058a3e..f1bd94aa 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -81,28 +81,39 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) { - TRACE("addCredential(COM) for %.16llx signed by %s issued to %s",com.networkId(),com.signedBy().toString().c_str(),com.issuedTo().toString().c_str()); - if (_com == com) + if (_com == com) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); return 0; + } const int vr = com.verify(RR); - if ((vr == 0)&&(com.timestamp().first > _com.timestamp().first)) - _com = com; + if (vr == 0) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); + if (com.timestamp().first > _com.timestamp().first) + _com = com; + } else { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); + } return vr; } int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) { TState *t = _tags.get(tag.id()); - if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) + if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) { + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); return 0; + } const int vr = tag.verify(RR); if (vr == 0) { + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); if (!t) t = &(_tags[tag.id()]); if (t->tag.timestamp() <= tag.timestamp()) { t->lastReceived = RR->node->now(); t->tag = tag; } + } else { + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (%d)",tag.issuedTo().toString().c_str(),tag.networkId(),vr); } return vr; } @@ -110,10 +121,13 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap) { std::map::iterator c(_caps.find(cap.id())); - if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) + if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) { + TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); return 0; + } const int vr = cap.verify(RR); if (vr == 0) { + TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); if (c == _caps.end()) { CState &c2 = _caps[cap.id()]; c2.lastReceived = RR->node->now(); @@ -122,6 +136,8 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap c->second.lastReceived = RR->node->now(); c->second.cap = cap; } + } else { + TRACE("addCredential(Capability) for %s on %.16llx REJECTED (%d)",cap.issuedTo().toString().c_str(),cap.networkId(),vr); } return vr; } -- cgit v1.2.3 From c476285bd638da01e0297c76951609c70f4ab3cf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 16:16:39 -0700 Subject: Harden PUSH_DIRECT_PATHS and simplify things by only doing it on receive when hops>0 and trust has been established. --- node/IncomingPacket.cpp | 78 ++++++++++++---------- node/Peer.cpp | 171 ++++++++++++++++++++++++------------------------ node/Peer.hpp | 18 ++--- node/Switch.cpp | 5 -- 4 files changed, 135 insertions(+), 137 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 1aecfdb7..8faa62fb 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -88,7 +88,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(_localAddress,_remoteAddress,hops(),packetId(),v,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; case Packet::VERB_HELLO: return _doHELLO(RR,peer); @@ -172,7 +172,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr default: break; } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -339,7 +339,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -461,7 +461,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -505,7 +505,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -527,8 +527,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< } else if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) { const uint64_t now = RR->node->now(); peer->sendHELLO(_localAddress,atAddr,now,2); // send low-TTL packet to 'open' local NAT(s) - if (!peer->pushDirectPaths(_localAddress,atAddr,now,true)) - peer->sendHELLO(_localAddress,atAddr,now); + peer->sendHELLO(_localAddress,atAddr,now); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -540,7 +539,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),with.toString().c_str()); } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -555,19 +554,17 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { if (!network->isAllowed(peer)) { TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id()); - return true; - } - - const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); - const MAC sourceMac(peer->address(),network->id()); - const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0)) { - RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,false); + } else { + const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); + const MAC sourceMac(peer->address(),network->id()); + const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; + const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; + if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0)) + RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true); } } - - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP); } else { TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } @@ -595,6 +592,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

isAllowed(peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -608,6 +606,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -616,24 +615,24 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

learnBridgeRoute(from,peer->address()); } else { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); - - if (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { + if (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); - } - } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + } } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } @@ -654,7 +653,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true); RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); - peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -673,7 +672,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared RR->mc->add(now,nwid,group,peer->address()); } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -722,7 +721,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -740,7 +739,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); if (RR->localNetworkController) { NetworkConfig *netconf = new NetworkConfig(); @@ -899,7 +898,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -929,6 +928,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // that cert might be what we needed. if (!network->isAllowed(peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -955,10 +955,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -967,6 +969,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } @@ -990,9 +993,11 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } } - } // else ignore -- not a member of this network - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); + } else { + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + } } catch ( ... ) { TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -1007,6 +1012,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha // First, subject this to a rate limit if (!peer->shouldRespondToDirectPathPush(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1069,7 +1075,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha ptr += addrLen; } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -1113,6 +1119,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } vlf += signatureLength; @@ -1129,10 +1136,12 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt SharedPtr network(RR->node->network(originatorCredentialNetworkId)); if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } @@ -1203,7 +1212,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt } } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -1248,6 +1257,7 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S } RR->node->postCircuitTestReport(&report); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } @@ -1308,7 +1318,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const break; } - peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP); + peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false); } else { TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); } diff --git a/node/Peer.cpp b/node/Peer.cpp index 77e1d0b5..7691408e 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -66,7 +66,8 @@ void Peer::received( uint64_t packetId, Packet::Verb verb, uint64_t inRePacketId, - Packet::Verb inReVerb) + Packet::Verb inReVerb, + const bool trustEstablished) { #ifdef ZT_ENABLE_CLUSTER bool suboptimalPath = false; @@ -184,6 +185,8 @@ void Peer::received( } } + } else if (trustEstablished) { + _pushDirectPaths(localAddr,remoteAddr,now); } if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) { @@ -241,90 +244,6 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) return false; } -bool Peer::pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now,bool force) -{ -#ifdef ZT_ENABLE_CLUSTER - // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection - if (RR->cluster) - return false; -#endif - - if (!force) { - if ((now - _lastDirectPathPushSent) < ZT_DIRECT_PATH_PUSH_INTERVAL) - return false; - else _lastDirectPathPushSent = now; - } - - std::vector pathsToPush; - - std::vector dps(RR->node->directPaths()); - for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) - pathsToPush.push_back(*i); - - std::vector sym(RR->sa->getSymmetricNatPredictions()); - for(unsigned long i=0,added=0;inode->prng() % sym.size()]); - if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { - pathsToPush.push_back(tmp); - if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) - break; - } - } - if (pathsToPush.empty()) - return false; - -#ifdef ZT_TRACE - { - std::string ps; - for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { - if (ps.length() > 0) - ps.push_back(','); - ps.append(p->toString()); - } - TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); - } -#endif - - std::vector::const_iterator p(pathsToPush.begin()); - while (p != pathsToPush.end()) { - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); - outp.addSize(2); // leave room for count - - unsigned int count = 0; - while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { - uint8_t addressType = 4; - switch(p->ss_family) { - case AF_INET: - break; - case AF_INET6: - addressType = 6; - break; - default: // we currently only push IP addresses - ++p; - continue; - } - - outp.append((uint8_t)0); // no flags - outp.append((uint16_t)0); // no extensions - outp.append(addressType); - outp.append((uint8_t)((addressType == 4) ? 6 : 18)); - outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); - outp.append((uint16_t)p->port()); - - ++count; - ++p; - } - - if (count) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); - outp.armor(_key,true); - RR->node->putPacket(localAddr,toAddress,outp.data(),outp.size(),0); - } - } - - return true; -} - bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) { unsigned int np = _numPaths; @@ -453,4 +372,86 @@ Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily) return bestPath; } +bool Peer::_pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now) +{ +#ifdef ZT_ENABLE_CLUSTER + // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection + if (RR->cluster) + return false; +#endif + + if ((now - _lastDirectPathPushSent) < ZT_DIRECT_PATH_PUSH_INTERVAL) + return false; + else _lastDirectPathPushSent = now; + + std::vector pathsToPush; + + std::vector dps(RR->node->directPaths()); + for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) + pathsToPush.push_back(*i); + + std::vector sym(RR->sa->getSymmetricNatPredictions()); + for(unsigned long i=0,added=0;inode->prng() % sym.size()]); + if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { + pathsToPush.push_back(tmp); + if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) + break; + } + } + if (pathsToPush.empty()) + return false; + +#ifdef ZT_TRACE + { + std::string ps; + for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { + if (ps.length() > 0) + ps.push_back(','); + ps.append(p->toString()); + } + TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); + } +#endif + + std::vector::const_iterator p(pathsToPush.begin()); + while (p != pathsToPush.end()) { + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); + outp.addSize(2); // leave room for count + + unsigned int count = 0; + while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { + uint8_t addressType = 4; + switch(p->ss_family) { + case AF_INET: + break; + case AF_INET6: + addressType = 6; + break; + default: // we currently only push IP addresses + ++p; + continue; + } + + outp.append((uint8_t)0); // no flags + outp.append((uint16_t)0); // no extensions + outp.append(addressType); + outp.append((uint8_t)((addressType == 4) ? 6 : 18)); + outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); + outp.append((uint16_t)p->port()); + + ++count; + ++p; + } + + if (count) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); + outp.armor(_key,true); + RR->node->putPacket(localAddr,toAddress,outp.data(),outp.size(),0); + } + } + + return true; +} + } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 200c5ac4..a6940737 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -104,6 +104,7 @@ public: * @param verb Packet verb * @param inRePacketId Packet ID in reply to (default: none) * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP) + * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established */ void received( const InetAddress &localAddr, @@ -111,8 +112,9 @@ public: unsigned int hops, uint64_t packetId, Packet::Verb verb, - uint64_t inRePacketId = 0, - Packet::Verb inReVerb = Packet::VERB_NOP); + uint64_t inRePacketId, + Packet::Verb inReVerb, + const bool trustEstablished); /** * Get the current best direct path to this peer @@ -192,17 +194,6 @@ public: */ bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); - /** - * Push direct paths back to self if we haven't done so in the configured timeout - * - * @param localAddr Local address - * @param toAddress Remote address to send push to (usually from path) - * @param now Current time - * @param force If true, push regardless of rate limit - * @return True if something was actually sent - */ - bool pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now,bool force); - /** * @return All known direct paths to this peer (active or inactive) */ @@ -407,6 +398,7 @@ private: void _doDeadPathDetection(Path &p,const uint64_t now); Path *_getBestPath(const uint64_t now); Path *_getBestPath(const uint64_t now,int inetAddressFamily); + bool _pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now); unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; diff --git a/node/Switch.cpp b/node/Switch.cpp index f6e4d1ab..546c9157 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -761,11 +761,6 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) return false; } - if (relay) { - peer->pushDirectPaths(viaPath->localAddress(),viaPath->address(),now,false); - viaPath->sent(now); - } - Packet tmp(packet); unsigned int chunkSize = std::min(tmp.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); -- cgit v1.2.3 From e52c2c41ecab0410f926965fd433da6eb9cd4039 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 17:24:35 -0700 Subject: Add a circuit breaker to prevent too many credentials from being stored per member. --- node/Membership.cpp | 30 +++++++++++++++++++++++++++++- node/Membership.hpp | 2 ++ 2 files changed, 31 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index f1bd94aa..8bf5b784 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -106,8 +106,24 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) const int vr = tag.verify(RR); if (vr == 0) { TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (!t) + if (!t) { + while (_tags.size() >= ZT_MAX_NETWORK_TAGS) { + uint32_t oldest = 0; + uint64_t oldestLastReceived = 0xffffffffffffffffULL; + uint32_t *i = (uint32_t *)0; + TState *ts = (TState *)0; + Hashtable::Iterator tsi(_tags); + while (tsi.next(i,ts)) { + if (ts->lastReceived < oldestLastReceived) { + oldestLastReceived = ts->lastReceived; + oldest = *i; + } + } + if (oldestLastReceived != 0xffffffffffffffffULL) + _tags.erase(oldest); + } t = &(_tags[tag.id()]); + } if (t->tag.timestamp() <= tag.timestamp()) { t->lastReceived = RR->node->now(); t->tag = tag; @@ -129,6 +145,18 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap if (vr == 0) { TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); if (c == _caps.end()) { + while (_caps.size() >= ZT_MAX_NETWORK_CAPABILITIES) { + std::map::iterator oldest; + uint64_t oldestLastReceived = 0xffffffffffffffffULL; + for(std::map::iterator i(_caps.begin());i!=_caps.end();++i) { + if (i->second.lastReceived < oldestLastReceived) { + oldestLastReceived = i->second.lastReceived; + oldest = i; + } + } + if (oldestLastReceived != 0xffffffffffffffffULL) + _caps.erase(oldest); + } CState &c2 = _caps[cap.id()]; c2.lastReceived = RR->node->now(); c2.cap = cap; diff --git a/node/Membership.hpp b/node/Membership.hpp index dc525483..bb356902 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -46,6 +46,8 @@ class RuntimeEnvironment; * * This is kind of analogous to a join table between Peer and Network. It is * presently held by the Network object for each participating Peer. + * + * This is not thread safe. It must be locked externally. */ class Membership { -- cgit v1.2.3 From 347ebcd899cb7182895a5d28e2be6159167e455c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 17:48:13 -0700 Subject: Set trust flag in network controllers if remote query is accepted to allow NATed network controllers to better traverse. --- node/IncomingPacket.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 8faa62fb..b74f7c8b 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -739,7 +739,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); + bool netconfOk = false; if (RR->localNetworkController) { NetworkConfig *netconf = new NetworkConfig(); @@ -747,6 +747,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,*netconf)) { case NetworkController::NETCONF_QUERY_OK: { + netconfOk = true; Dictionary *dconf = new Dictionary(); try { if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { @@ -817,6 +818,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.armor(peer->key(),true); RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); } + + peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what()); -- cgit v1.2.3 From cd3683f2bae5e706b5afdf40eee2b5f486f5aedb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 17:50:51 -0700 Subject: Fix a missing receive(). --- node/IncomingPacket.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b74f7c8b..0804f04a 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -841,6 +841,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons network->requestConfiguration(); } else { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); return true; } @@ -851,6 +852,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons ptr += 13; } } + + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); } -- cgit v1.2.3 From 584228b2b5ff91a3db4699174bbefe1540d4ce59 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 17:56:35 -0700 Subject: Dead code removal, and get rid of reliable() because we will no longer make that distinction. --- node/Path.hpp | 42 ------------------------------------------ node/Peer.cpp | 4 +--- 2 files changed, 1 insertion(+), 45 deletions(-) (limited to 'node') diff --git a/node/Path.hpp b/node/Path.hpp index ecf4be24..ca5dd98f 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -247,16 +247,6 @@ public: return score; } - /** - * @return True if path is considered reliable (no NAT keepalives etc. are needed) - */ - inline bool reliable() const throw() - { - if ((_addr.ss_family == AF_INET)||(_addr.ss_family == AF_INET6)) - return ((_ipScope != InetAddress::IP_SCOPE_GLOBAL)&&(_ipScope != InetAddress::IP_SCOPE_PSEUDOPRIVATE)); - return true; - } - /** * @return True if address is non-NULL */ @@ -313,38 +303,6 @@ public: */ inline void increaseProbation() { ++_probation; } - template - inline void serialize(Buffer &b) const - { - b.append((uint8_t)2); // version - b.append((uint64_t)_lastSend); - b.append((uint64_t)_lastPing); - b.append((uint64_t)_lastKeepalive); - b.append((uint64_t)_lastReceived); - _addr.serialize(b); - _localAddress.serialize(b); - b.append((uint16_t)_flags); - b.append((uint16_t)_probation); - } - - template - inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) - { - unsigned int p = startAt; - if (b[p++] != 2) - throw std::invalid_argument("invalid serialized Path"); - _lastSend = b.template at(p); p += 8; - _lastPing = b.template at(p); p += 8; - _lastKeepalive = b.template at(p); p += 8; - _lastReceived = b.template at(p); p += 8; - p += _addr.deserialize(b,p); - p += _localAddress.deserialize(b,p); - _flags = b.template at(p); p += 2; - _probation = b.template at(p); p += 2; - _ipScope = _addr.ipScope(); - return (p - startAt); - } - inline bool operator==(const Path &p) const { return ((p._addr == _addr)&&(p._localAddress == _localAddress)); } inline bool operator!=(const Path &p) const { return ((p._addr != _addr)||(p._localAddress != _localAddress)); } diff --git a/node/Peer.cpp b/node/Peer.cpp index 7691408e..01492be1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -230,13 +230,11 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) sendHELLO(p->localAddress(),p->address(),now); p->sent(now); p->pinged(now); - } else if ( ((now - std::max(p->lastSend(),p->lastKeepalive())) >= ZT_NAT_KEEPALIVE_DELAY) && (!p->reliable()) ) { + } else if ((now - std::max(p->lastSend(),p->lastKeepalive())) >= ZT_NAT_KEEPALIVE_DELAY) { //TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived()); _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads RR->node->putPacket(p->localAddress(),p->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf)); p->sentKeepalive(now); - } else { - //TRACE("no PING or NAT keepalive: addr==%s reliable==%d %llums/%llums send/receive inactivity",p->address().toString().c_str(),(int)p->reliable(),now - p->lastSend(),now - p->lastReceived()); } return true; } -- cgit v1.2.3 From 5eaf397a948923d3de68763c972be7ae8c83132d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 25 Aug 2016 13:31:23 -0700 Subject: Add a debug log feature in the filter, which only works if enabled in Network.cpp. --- controller/EmbeddedNetworkController.cpp | 6 ++ controller/README.md | 2 +- include/ZeroTierOne.h | 5 ++ node/Capability.hpp | 1 + node/Network.cpp | 94 +++++++++++++++++++++++--------- 5 files changed, 80 insertions(+), 28 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 713b7618..1899edb9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -147,6 +147,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["flags"] = (uint64_t)rule.v.fwd.flags; r["length"] = (uint64_t)rule.v.fwd.length; break; + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: + r["type"] = "ACTION_DEBUG_LOG"; + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; r["zt"] = Address(rule.v.zt).toString(); @@ -278,6 +281,9 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; + } else if (t == "ACTION_DEBUG_LOG") { + rule.t |= ZT_NETWORK_RULE_ACTION_DEBUG_LOG; + return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; diff --git a/controller/README.md b/controller/README.md index 189fcdbd..68b8cdb6 100644 --- a/controller/README.md +++ b/controller/README.md @@ -5,7 +5,7 @@ ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. -Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. Technically the JSON files under `controller.d` can also be edited in place, but we do not recommend doing this under a running controller since data loss or corruption might result. Also take care to keep JSON values of the correct types or data loss may also result. +Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, but they can be safely backed up or copied. ### Scalability and Reliability diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 4c0da76d..1d7620d4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -515,6 +515,11 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_ACTION_REDIRECT = 3, + /** + * Log if match and if rule debugging is enabled in the build, otherwise does nothing (for developers) + */ + ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 4, + // 32 to 127 reserved for match criteria /** diff --git a/node/Capability.hpp b/node/Capability.hpp index 0b352725..ce2eedc4 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -176,6 +176,7 @@ public: switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x7f)) { //case ZT_NETWORK_RULE_ACTION_DROP: //case ZT_NETWORK_RULE_ACTION_ACCEPT: + //case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: default: b.append((uint8_t)0); break; diff --git a/node/Network.cpp b/node/Network.cpp index ecbacd93..02f53b55 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -35,9 +35,13 @@ #include "Node.hpp" #include "Peer.hpp" +// Uncomment to enable ZT_NETWORK_RULE_ACTION_DEBUG_LOG rule output to STDOUT +#define ZT_RULES_ENGINE_DEBUGGING 1 + namespace ZeroTier { -#ifdef ZT_TRACE +#ifdef ZT_RULES_ENGINE_DEBUGGING +#define FILTER_TRACE(f,...) { Utils::snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) { switch(rt) { @@ -45,6 +49,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_ACTION_ACCEPT: return "ACTION_ACCEPT"; case ZT_NETWORK_RULE_ACTION_TEE: return "ACTION_TEE"; case ZT_NETWORK_RULE_ACTION_REDIRECT: return "ACTION_REDIRECT"; + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: return "ACTION_DEBUG_LOG"; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: return "MATCH_SOURCE_ZEROTIER_ADDRESS"; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: return "MATCH_DEST_ZEROTIER_ADDRESS"; case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; @@ -65,7 +70,9 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) default: return "BAD_RULE_TYPE"; } } -#endif // ZT_TRACE +#else +#define FILTER_TRACE(f,...) {} +#endif // ZT_RULES_ENGINE_DEBUGGING // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) @@ -96,9 +103,6 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } -//#define FILTER_TRACE TRACE -#define FILTER_TRACE(f,...) {} - // 0 == no match, -1 == match/drop, 1 == match/accept static int _doZtFilter( const RuntimeEnvironment *RR, @@ -127,6 +131,11 @@ static int _doZtFilter( // yields a 'match all' rule). uint8_t thisSetMatches = 1; +#ifdef ZT_RULES_ENGINE_DEBUGGING + std::vector dlog; + char dpbuf[1024]; +#endif + for(unsigned int rn=0;rn%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + ztSource.toString().c_str(), + ztDest.toString().c_str(), + (unsigned int)macSource[0], + (unsigned int)macSource[1], + (unsigned int)macSource[2], + (unsigned int)macSource[3], + (unsigned int)macSource[4], + (unsigned int)macSource[5], + (unsigned int)macDest[0], + (unsigned int)macDest[1], + (unsigned int)macDest[2], + (unsigned int)macDest[3], + (unsigned int)macDest[4], + (unsigned int)macDest[5], + (int)inbound, + (int)noRedirect, + frameLen, + etherType + ); + for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) + printf(" %s" ZT_EOL_S,m->c_str()); + dlog.clear(); + } +#endif // ZT_RULES_ENGINE_DEBUGGING + thisRuleMatches = 1; + thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation + break; // Rules --------------------------------------------------------------- // thisSetMatches is the binary AND of the result of all rules in a set case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - FILTER_TRACE("FILTER[%u] %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); + FILTER_TRACE("%u %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - FILTER_TRACE("FILTER[%u] %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); + FILTER_TRACE("%u %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: // NOT SUPPORTED YET - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: // NOT SUPPORTED YET - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - FILTER_TRACE("FILTER[%u] %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); + FILTER_TRACE("%u %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - FILTER_TRACE("FILTER[%u] %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); + FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: - FILTER_TRACE("FILTER[%u] %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); + FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); } else { @@ -222,7 +262,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); } else { @@ -230,7 +270,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); } else { @@ -238,7 +278,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - FILTER_TRACE("FILTER[%u] %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); + FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); } else { @@ -246,7 +286,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { @@ -257,7 +297,7 @@ static int _doZtFilter( } break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); } else if (etherType == ZT_ETHERTYPE_IPV6) { @@ -289,7 +329,7 @@ static int _doZtFilter( } break; } - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; @@ -308,14 +348,14 @@ static int _doZtFilter( } break; } - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; } else { - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; } } else { - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); + FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; } break; @@ -336,18 +376,18 @@ static int _doZtFilter( } } } - FILTER_TRACE("FILTER[%u] %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); + FILTER_TRACE("%u %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - FILTER_TRACE("FILTER[%u] %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); + FILTER_TRACE("%u %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { - FILTER_TRACE("FILTER[%u] %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); + FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); const Tag *lt = (const Tag *)0; for(unsigned int i=0;i> 7)); - FILTER_TRACE("FILTER[%u] %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); + FILTER_TRACE("%u %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); } return 0; -- cgit v1.2.3 From b5e0d014ab7fc96f8ebc3444cc1d2f3f0648c66f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 25 Aug 2016 16:08:40 -0700 Subject: Controller bug fixes --- controller/EmbeddedNetworkController.cpp | 208 +++++++++++++++++++------------ controller/EmbeddedNetworkController.hpp | 2 + node/Network.cpp | 30 +++++ 3 files changed, 158 insertions(+), 82 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1899edb9..23514448 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -127,7 +127,6 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; json r = json::object(); - r["not"] = ((rule.t & 0x80) != 0); switch((rule.t) & 0x7f) { case ZT_NETWORK_RULE_ACTION_DROP: r["type"] = "ACTION_DROP"; @@ -152,74 +151,91 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["not"] = ((rule.t & 0x80) != 0); r["zt"] = Address(rule.v.zt).toString(); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["not"] = ((rule.t & 0x80) != 0); r["zt"] = Address(rule.v.zt).toString(); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: r["type"] = "MATCH_VLAN_ID"; + r["not"] = ((rule.t & 0x80) != 0); r["vlanId"] = (uint64_t)rule.v.vlanId; break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: r["type"] = "MATCH_VLAN_PCP"; + r["not"] = ((rule.t & 0x80) != 0); r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: r["type"] = "MATCH_VLAN_DEI"; + r["not"] = ((rule.t & 0x80) != 0); r["vlanDei"] = (uint64_t)rule.v.vlanDei; break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: r["type"] = "MATCH_ETHERTYPE"; + r["not"] = ((rule.t & 0x80) != 0); r["etherType"] = (uint64_t)rule.v.etherType; break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; + r["not"] = ((rule.t & 0x80) != 0); Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; + r["not"] = ((rule.t & 0x80) != 0); Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: r["type"] = "MATCH_IPV4_SOURCE"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: r["type"] = "MATCH_IPV4_DEST"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: r["type"] = "MATCH_IPV6_SOURCE"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: r["type"] = "MATCH_IPV6_DEST"; + r["not"] = ((rule.t & 0x80) != 0); r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); break; case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; + r["not"] = ((rule.t & 0x80) != 0); r["ipTos"] = (uint64_t)rule.v.ipTos; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: r["type"] = "MATCH_IP_PROTOCOL"; + r["not"] = ((rule.t & 0x80) != 0); r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["not"] = ((rule.t & 0x80) != 0); r["start"] = (uint64_t)rule.v.port[0]; r["end"] = (uint64_t)rule.v.port[1]; break; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["not"] = ((rule.t & 0x80) != 0); r["start"] = (uint64_t)rule.v.port[0]; r["end"] = (uint64_t)rule.v.port[1]; break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; + r["not"] = ((rule.t & 0x80) != 0); Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); r["mask"] = tmp; Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); @@ -227,26 +243,31 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["not"] = ((rule.t & 0x80) != 0); r["start"] = (uint64_t)rule.v.frameSize[0]; r["end"] = (uint64_t)rule.v.frameSize[1]; break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: r["type"] = "MATCH_TAGS_SAMENESS"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["not"] = ((rule.t & 0x80) != 0); r["id"] = (uint64_t)rule.v.tag.id; r["value"] = (uint64_t)rule.v.tag.value; break; @@ -254,9 +275,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) return r; } -static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) +static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) { - if (r.is_object()) + if (!r.is_object()) return false; const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); @@ -494,7 +515,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( lrt = now; } - const json network(_readJson(_networkJP(nwid,false))); + json network(_readJson(_networkJP(nwid,false))); if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; @@ -635,14 +656,14 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( for(std::set

::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); - const json &v4AssignMode = network["v4AssignMode"]; - const json &v6AssignMode = network["v6AssignMode"]; - const json &ipAssignmentPools = network["ipAssignmentPools"]; - const json &routes = network["routes"]; - const json &rules = network["rules"]; - const json &capabilities = network["capabilities"]; - const json &memberCapabilities = member["capabilities"]; - const json &memberTags = member["tags"]; + json &v4AssignMode = network["v4AssignMode"]; + json &v6AssignMode = network["v6AssignMode"]; + json &ipAssignmentPools = network["ipAssignmentPools"]; + json &routes = network["routes"]; + json &rules = network["rules"]; + json &capabilities = network["capabilities"]; + json &memberCapabilities = member["capabilities"]; + json &memberTags = member["tags"]; if (rules.is_array()) { for(unsigned long i=0;i 0)&&(capabilities.is_array())) { - std::map< uint64_t,const json * > capsById; + std::map< uint64_t,json * > capsById; for(unsigned long i=0;iis_object())&&(cap->size() > 0)) { ZT_VirtualNetworkRule capr[ZT_MAX_CAPABILITY_RULES]; unsigned int caprc = 0; - auto caprj = (*cap)["rules"]; + json &caprj = (*cap)["rules"]; if ((caprj.is_array())&&(caprj.size() > 0)) { for(unsigned long j=0;j= ZT_MAX_CAPABILITY_RULES) @@ -688,7 +709,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if (memberTags.is_array()) { std::map< uint32_t,uint32_t > tagsById; for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) break; - auto route = routes[i]; - InetAddress t(_jS(route["target"],"")); - InetAddress v(_jS(route["via"],"")); - if ((t)&&(v)&&(t.ss_family == v.ss_family)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); - *(reinterpret_cast(&(r->target))) = t; - *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; + json &route = routes[i]; + json &target = route["target"]; + json &via = route["via"]; + if (target.is_string()) { + const InetAddress t(target.get()); + InetAddress v; + if (via.is_string()) v.fromString(via.get()); + if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { + ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + *(reinterpret_cast(&(r->target))) = t; + if (v.ss_family == t.ss_family) + *(reinterpret_cast(&(r->via))) = v; + ++nc.routeCount; + } } } } - if (v6AssignMode.is_object()) { + const bool noAutoAssignIps = _jB(member["noAutoAssignIps"],false); + + if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; @@ -730,9 +759,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( bool haveManagedIpv4AutoAssignment = false; bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count - json ipAssignments = member["ipAssignments"]; + json ipAssignments = member["ipAssignments"]; // we want to make a copy if (ipAssignments.is_array()) { for(unsigned long i=0;i(&ipRangeStart)->sin_addr.s_addr)); - uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); - if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) + InetAddress ipRangeStartIA(_jS(pool["ipRangeStart"],"")); + InetAddress ipRangeEndIA(_jS(pool["ipRangeEnd"],"")); + if ( (ipRangeStartIA.ss_family == AF_INET) && (ipRangeEndIA.ss_family == AF_INET) ) { + uint32_t ipRangeStart = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeStartIA)->sin_addr.s_addr)); + uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); + if ((ipRangeEnd < ipRangeStart)||(ipRangeStart == 0)) continue; uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; // Start with the LSB of the member's address uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { + for(uint32_t k=ipRangeStart,trialCount=0;((k<=ipRangeEnd)&&(trialCount < 1000));++k,++trialCount) { uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; ++ipTrialCounter; if ((ip & 0x000000ff) == 0x000000ff) continue; // don't allow addresses that end in .255 // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; + int routedNetmaskBits = -1; for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { @@ -855,9 +886,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - InetAddress ip4(Utils::hton(ip),0); - // If it's routed, then try to claim and assign it and if successful end loop + const InetAddress ip4(Utils::hton(ip),0); if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; @@ -1048,9 +1078,18 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json b; try { b = json::parse(body); - if (!b.is_object()) + if (!b.is_object()) { + responseBody = "{ \"message\": \"body is not a JSON object\" }"; + responseContentType = "application/json"; return 400; + } + } catch (std::exception &exc) { + responseBody = std::string("{ \"message\": \"body JSON is invalid: ") + exc.what() + "\" }"; + responseContentType = "application/json"; + return 400; } catch ( ... ) { + responseBody = "{ \"message\": \"body JSON is invalid\" }"; + responseContentType = "application/json"; return 400; } const uint64_t now = OSUtils::now(); @@ -1143,6 +1182,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } } } catch ( ... ) { + responseBody = "{ \"message\": \"exception while processing parameters in JSON body\" }"; + responseContentType = "application/json"; return 400; } @@ -1204,6 +1245,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!test->hopCount) { ::free((void *)test); + responseBody = "{ \"message\": \"a test must contain at least one hop\" }"; + responseContentType = "application/json"; return 400; } @@ -1258,19 +1301,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); if (b.count("v4AssignMode")) { - auto nv4m = network["v4AssignMode"]; + json &nv4m = network["v4AssignMode"]; if (!nv4m.is_object()) nv4m = json::object(); if (b["v4AssignMode"].is_string()) { // backward compatibility nv4m["zt"] = (_jS(b["v4AssignMode"],"") == "zt"); } else if (b["v4AssignMode"].is_object()) { - auto v4m = b["v4AssignMode"]; + json &v4m = b["v4AssignMode"]; if (v4m.count("zt")) nv4m["zt"] = _jB(v4m["zt"],false); } if (!nv4m.count("zt")) nv4m["zt"] = false; } if (b.count("v6AssignMode")) { - auto nv6m = network["v6AssignMode"]; + json &nv6m = network["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (b["v6AssignMode"].is_string()) { // backward compatibility std::vector v6m(Utils::split(_jS(b["v6AssignMode"],"").c_str(),",","","")); @@ -1285,7 +1328,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( nv6m["6plane"] = true; } } else if (b["v6AssignMode"].is_object()) { - auto v6m = b["v6AssignMode"]; + json &v6m = b["v6AssignMode"]; if (v6m.count("rfc4193")) nv6m["rfc4193"] = _jB(v6m["rfc4193"],false); if (v6m.count("zt")) nv6m["rfc4193"] = _jB(v6m["zt"],false); if (v6m.count("6plane")) nv6m["rfc4193"] = _jB(v6m["6plane"],false); @@ -1296,61 +1339,64 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } if (b.count("routes")) { - auto rts = b["routes"]; + json &rts = b["routes"]; if (rts.is_array()) { + json nrts = json::array(); for(unsigned long i=0;i()); + InetAddress v; + if (via.is_string()) v.fromString(via.get()); + if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.netmaskBitsValid()) ) { + json tmp; + tmp["target"] = t.toString(); + if (v.ss_family == t.ss_family) + tmp["via"] = v.toIpString(); + else tmp["via"] = json(); + nrts.push_back(tmp); + } } } } + network["routes"] = nrts; } } if (b.count("ipAssignmentPools")) { - auto ipp = b["ipAssignmentPools"]; + json &ipp = b["ipAssignmentPools"]; if (ipp.is_array()) { + json nipp = json::array(); for(unsigned long i=0;i ncaps; for(unsigned long i=0;i Date: Thu, 25 Aug 2016 18:21:20 -0700 Subject: Fix chicken or egg problem in tags, and better filter debug instrumentation. --- controller/EmbeddedNetworkController.cpp | 2 +- include/ZeroTierOne.h | 6 +- node/Membership.cpp | 25 ++++-- node/Membership.hpp | 10 ++- node/Network.cpp | 136 ++++++++++++++++--------------- 5 files changed, 96 insertions(+), 83 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 8f013464..45af30be 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -50,7 +50,7 @@ using json = nlohmann::json; #define ZT_NETCONF_CONTROLLER_API_VERSION 3 // Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 16 +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 32 // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 1d7620d4..a3d82adb 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -107,14 +107,14 @@ extern "C" { #define ZT_MAX_NETWORK_RULES 1024 /** - * Maximum number of per-node capabilities per network + * Maximum number of per-member capabilities per network */ #define ZT_MAX_NETWORK_CAPABILITIES 128 /** - * Maximum number of per-node tags per network + * Maximum number of per-member tags per network */ -#define ZT_MAX_NETWORK_TAGS 128 +#define ZT_MAX_NETWORK_TAGS 16 /** * Maximum number of direct network paths to a given peer diff --git a/node/Membership.cpp b/node/Membership.cpp index 8bf5b784..7ced9dcf 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -28,8 +28,12 @@ namespace ZeroTier { -bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount) +bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) { + if ((now - _lastPushAttempt) < 1000ULL) + return false; + _lastPushAttempt = now; + try { Buffer capsAndTags; @@ -50,27 +54,29 @@ bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint unsigned int appendedTags = 0; const unsigned int tagCountPos = capsAndTags.size(); capsAndTags.addSize(2); - for(unsigned int i=0;iid()); + for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { if ((capsAndTags.size() + sizeof(Tag)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) break; - tags[i]->serialize(capsAndTags); + nconf.tags[i].serialize(capsAndTags); ts->lastPushed = now; ++appendedTags; } } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - if ( ((com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)) || (appendedCaps) || (appendedTags) ) { + const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); + if ( (needCom) || (appendedCaps) || (appendedTags) ) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - if (com) - com.serialize(outp); + if (needCom) { + nconf.com.serialize(outp); + _lastPushedCom = now; + } outp.append((uint8_t)0x00); outp.append(capsAndTags.data(),capsAndTags.size()); outp.compress(); RR->sw->send(outp,true); - _lastPushedCom = now; return true; } } catch ( ... ) { @@ -88,8 +94,9 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe const int vr = com.verify(RR); if (vr == 0) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); - if (com.timestamp().first > _com.timestamp().first) + if (com.timestamp().first > _com.timestamp().first) { _com = com; + } } else { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); } diff --git a/node/Membership.hpp b/node/Membership.hpp index bb356902..580aa529 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -107,6 +107,7 @@ public: friend class CapabilityIterator; Membership() : + _lastPushAttempt(0), _lastPushedCom(0), _blacklistBefore(0), _com(), @@ -124,13 +125,11 @@ public: * @param RR Runtime environment * @param now Current time * @param peerAddress Address of member peer - * @param com My network certificate of membership (if any) (not the one here, but ours -- in NetworkConfig) + * @param nconf My network config * @param cap Capability to send or 0 if none - * @param tags Tags that this peer might need - * @param tagCount Number of tag IDs * @return True if we pushed something */ - bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const CertificateOfMembership &com,const Capability *cap,const Tag **tags,const unsigned int tagCount); + bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); /** * @param nconf Our network config @@ -270,6 +269,9 @@ public: } private: + // Last time we checked if credential push was needed + uint64_t _lastPushAttempt; + // Last time we pushed our COM to this peer uint64_t _lastPushedCom; diff --git a/node/Network.cpp b/node/Network.cpp index 5293e2f0..1267f99c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -143,7 +143,7 @@ static int _doZtFilter( const Address &ztDest, const MAC &macSource, const MAC &macDest, - const uint8_t *frameData, + const uint8_t *const frameData, const unsigned int frameLen, const unsigned int etherType, const unsigned int vlanId, @@ -151,11 +151,9 @@ static int _doZtFilter( const unsigned int ruleCount, const Tag *localTags, const unsigned int localTagCount, - const uint32_t *remoteTagIds, - const uint32_t *remoteTagValues, - const unsigned int remoteTagCount, - const Tag **relevantLocalTags, // pointer array must be at least [localTagCount] in size - unsigned int &relevantLocalTagCount) + const uint32_t *const remoteTagIds, + const uint32_t *const remoteTagValues, + const unsigned int remoteTagCount) { // For each set of rules we start by assuming that they match (since no constraints // yields a 'match all' rule). @@ -168,30 +166,22 @@ static int _doZtFilter( for(unsigned int rn=0;rn %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.zt,ztSource.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - FILTER_TRACE("%u %s param0=%.10llx",rn,_rtn(rt),rules[rn].v.zt); thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); + FILTER_TRACE("%u %s %c %.10llx==%.10llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.zt,ztDest.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanId); thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanId,(unsigned int)vlanId,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: // NOT SUPPORTED YET - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanPcp); thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanPcp,0,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: // NOT SUPPORTED YET - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.vlanDei); thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanDei,0,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - FILTER_TRACE("%u %s param0=%u etherType=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.etherType,etherType); thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); + FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macSource.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: - FILTER_TRACE("%u %s param0=%.12llx",rn,_rtn(rt),rules[rn].v.mac); thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); + FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macDest.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str(),InetAddress((const void *)(frameData + 12),4,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv4] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str(),InetAddress((const void *)(frameData + 16),4,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv4] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str(),InetAddress((const void *)(frameData + 8),16,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - FILTER_TRACE("%u %s param0=%s",rn,_rtn(rt),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str()); if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); + FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str(),InetAddress((const void *)(frameData + 24),16,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipTos); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((frameData[1] & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); + FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((trafficClass & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.ipProtocol); if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); + FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipProtocol,(unsigned int)frameData[9],(unsigned int)thisRuleMatches); } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto); + FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipProtocol,proto,(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: @@ -359,8 +362,9 @@ static int _doZtFilter( } break; } - FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto==%u etherType=%u (IPv4)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,(unsigned int)frameData[9],etherType); - thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + + thisRuleMatches = (p >= 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + FILTER_TRACE("%u %s %c (IPv4) %d in %d-%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),p,(int)rules[rn].v.port[0],(int)rules[rn].v.port[1],(unsigned int)thisRuleMatches); } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { @@ -378,15 +382,15 @@ static int _doZtFilter( } break; } - FILTER_TRACE("%u %s param0=%u param1=%u port==%u proto=%u etherType=%u (IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],p,proto,etherType); thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; + FILTER_TRACE("%u %s %c (IPv6) %d in %d-%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),p,(int)rules[rn].v.port[0],(int)rules[rn].v.port[1],(unsigned int)thisRuleMatches); } else { - FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (IPv6 parse failed)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { - FILTER_TRACE("%u %s param0=%u param1=%u port=0 proto=0 etherType=%u (not IPv4 or IPv6)",rn,_rtn(rt),(unsigned int)rules[rn].v.port[0],(unsigned int)rules[rn].v.port[1],etherType); thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { @@ -406,18 +410,17 @@ static int _doZtFilter( } } } - FILTER_TRACE("%u %s param0=%.16llx param1=%.16llx actual=%.16llx",rn,_rtn(rt),rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],cf); thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); + FILTER_TRACE("%u %s %c (%.16llx & %.16llx)==%.16llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - FILTER_TRACE("%u %s param0=%u param1=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1]); thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); + FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { - FILTER_TRACE("%u %s param0=%u",rn,_rtn(rt),(unsigned int)rules[rn].v.tag.value); const Tag *lt = (const Tag *)0; for(unsigned int i=0;i 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { const uint32_t *rtv = (const uint32_t *)0; for(unsigned int i=0;i 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { if (rt == ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS) { const uint32_t sameness = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); thisRuleMatches = (uint8_t)(sameness <= rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u sameness:%u <= %u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,sameness,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { - thisRuleMatches = (uint8_t)((lt->value() & *rtv) <= rules[rn].v.tag.value); + thisRuleMatches = (uint8_t)((lt->value() & *rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) { - thisRuleMatches = (uint8_t)((lt->value() | *rtv) <= rules[rn].v.tag.value); + thisRuleMatches = (uint8_t)((lt->value() | *rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { - thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) <= rules[rn].v.tag.value); + thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } - if (thisRuleMatches) { - relevantLocalTags[relevantLocalTagCount++] = lt; - } } } } break; + + default: continue; } // thisSetMatches remains true if the current rule matched (or did NOT match if NOT bit is set) - thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t & 0x80) >> 7)); - - FILTER_TRACE("%u %s/%u thisRuleMatches==%u thisSetMatches==%u",rn,_rtn(rt),(unsigned int)rt,(unsigned int)thisRuleMatches,(unsigned int)thisSetMatches); + thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } return 0; @@ -543,31 +549,32 @@ bool Network::filterOutgoingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; - const Tag *relevantLocalTags[ZT_MAX_NETWORK_TAGS]; - unsigned int relevantLocalTagCount = 0; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: + if (ztDest) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return false; case 1: if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,(const Capability *)0,relevantLocalTags,relevantLocalTagCount); + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return true; } for(unsigned int c=0;c<_config.capabilityCount;++c) { - relevantLocalTagCount = 0; - switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: + if (ztDest) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return false; case 1: if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config.com,&(_config.capabilities[c]),relevantLocalTags,relevantLocalTagCount); + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,&(_config.capabilities[c])); return true; } } @@ -587,15 +594,13 @@ bool Network::filterIncomingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; - const Tag *relevantLocalTags[ZT_MAX_NETWORK_TAGS]; - unsigned int relevantLocalTagCount = 0; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: return false; case 1: @@ -605,8 +610,7 @@ bool Network::filterIncomingPacket( Membership::CapabilityIterator mci(m); const Capability *c; while ((c = mci.next(_config))) { - relevantLocalTagCount = 0; - switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,relevantLocalTags,relevantLocalTagCount)) { + switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { case -1: return false; case 1: @@ -1030,7 +1034,7 @@ void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std:: { Membership *m = _memberships.get(peer->address()); if (m) - m->sendCredentialsIfNeeded(RR,RR->node->now(),peer->address(),_config.com,(const Capability *)0,(const Tag **)0,0); + m->sendCredentialsIfNeeded(RR,RR->node->now(),peer->address(),_config,(const Capability *)0); } Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); -- cgit v1.2.3 From ded5a53a6ce5f6de2f5ebfc76f5d1ca68edc605b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 10:38:43 -0700 Subject: Documentation updates, add rules engine revision to network config request meta-data. --- controller/README.md | 36 ++++++++++++++++++++++++++++------- include/ZeroTierOne.h | 7 ++++++- node/Network.cpp | 51 ++++++++++++++++---------------------------------- node/NetworkConfig.hpp | 2 ++ one.cpp | 6 ++++++ 5 files changed, 59 insertions(+), 43 deletions(-) (limited to 'node') diff --git a/controller/README.md b/controller/README.md index 68b8cdb6..1eb0ca0d 100644 --- a/controller/README.md +++ b/controller/README.md @@ -1,15 +1,33 @@ Network Controller Microservice ====== -ZeroTier's 16-digit network IDs are really just a concatenation of the 10-digit ZeroTier address of a network controller followed by a 6-digit (24-bit) network number on that controller. Fans of software defined networking will recognize this as a spin on the familiar [separation of data plane and control plane](http://sdntutorials.com/difference-between-control-plane-and-data-plane/) SDN design pattern. +Every ZeroTier virtual network has a *network controller*. This is our reference implementation and is the same one we use to power our own hosted services at [my.zerotier.com](https://my.zerotier.com/). Network controllers act as configuration servers and certificate authorities for the members of networks. Controllers are located on the network by simply parsing out the first 10 digits of a network's 16-digit network ID: these are the address of the controller. -This code implements the *node/NetworkController.hpp* interface to provide an embedded microservice configurable via the same local HTTP control plane as ZeroTier One iteself. It is built by default in ZeroTier One in desktop and server builds. This is the same code we use to run [my.zerotier.com](https://my.zerotier.com/), which is a web UI and API that runs in front of a pool of controllers. +As of ZeroTier One version 1.2.0 this code is included in normal builds for desktop, laptop, and server (Linux, etc.) targets, allowing any device to create virtual networks without having to be rebuilt from source with special flags to enable this feature. While this does offer a convenient way to create ad-hoc networks or experiment, we recommend running a dedicated controller somewhere secure and stable for any "serious" use case. -Data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, tar'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, but they can be safely backed up or copied. +Controller data is stored in JSON format under `controller.d` in the ZeroTier working directory. It can be copied, rsync'd, placed in `git`, etc. The files under `controller.d` should not be modified in place while the controller is running or data loss may result, and if they are edited directly take care not to save corrupt JSON since that can also lead to data loss when the controller is restarted. Going through the API is strongly preferred to directly modifying these files. + +### Upgrading from Older Versions + +Older versions of this code used a SQLite database instead of in-filesystem JSON. A migration utility called `migrate-sqlite` is included here and *must* be used to migrate this data to the new format. If the controller is started with an old `controller.db` in its working directory it will terminate after printing an error to *stderr*. This is done to prevent "surprises" for those running DIY controllers using the old code. + +The migration tool is written in nodeJS and can be used like this: + + cd migrate-sqlite + npm install + node migrate-sqlite.js + +You may need to `sudo node ...` if the ZeroTier working directory is owned by root. + +This code will dump the contents of any `controller.db` in the ZeroTier working directory and recreate its data in the form of JSON objects under `controller.d`. The old `controller.db` will then be renamed to `controller.db.migrated` and the controller will start normally. + +After migrating make sure that the contents of `controller.d` are owned and writable by the user that will be running the ZeroTier controller process! (Usually this is root but controllers that don't also join networks are sometimes run as unprivileged users.) + +If you don't have nodeJS on the machine running ZeroTier it is perfectly fine to just copy `controller.db` to a directory on another machine and run the migration tool there. After running your migration the contents of `controller.d` can be tar'd up and copied back over to the controller. Just remember to rename or remove `controller.db` on the controller machine afterwords so the controller will start. ### Scalability and Reliability -Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. +Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. Since the controller uses the filesystem as its data store we recommend fast filesystems and fast SSD drives for heavily loaded controllers. Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Many modern orchestration tools have built-in support for this. It would also be possible in theory to run controllers on a replicated or distributed filesystem, but we haven't tested this yet. @@ -154,6 +172,7 @@ The entry types and their additional fields are: | `ACTION_ACCEPT` | Accept any packets matching this rule | (none) | | `ACTION_TEE` | Send a copy of this packet to a node (rule parsing continues) | `zt` | | `ACTION_REDIRECT` | Redirect this packet to another node | `zt` | +| `ACTION_DEBUG_LOG` | Output debug info on match (if built with rules engine debug) | (none) | | `MATCH_SOURCE_ZEROTIER_ADDRESS` | Match VL1 ZeroTier address of packet sender. | `zt` | | `MATCH_DEST_ZEROTIER_ADDRESS` | Match VL1 ZeroTier address of recipient | `zt` | | `MATCH_ETHERTYPE` | Match Ethernet frame type | `etherType` | @@ -174,10 +193,12 @@ The entry types and their additional fields are: | `MATCH_TAGS_BITWISE_OR` | Match if both sides' tags OR to value | `id`,`value` | | `MATCH_TAGS_BITWISE_XOR` | Match if both sides` tags XOR to value | `id`,`value` | -Notes: +Important notes about rules engine behavior: - * `MATCH_DEST_ZEROTIER_ADDRESS` does not work for multicast packets since these have many and varying destinations and can take circuitous routes. Use MAC rules instead. - * IPv4 and IPv6 rules do not match for frames that are not IPv4 or IPv6 respectively. + * IPv4 and IPv6 IP address rules do not match for frames that are not IPv4 or IPv6 respectively. + * `ACTION_DEBUG_LOG` is a no-op on nodes not built with `ZT_RULES_ENGINE_DEBUGGING` enabled (see Network.cpp). If that is enabled nodes will dump a trace of rule evaluation results to *stdout* when this action is encountered but will otherwise keep evaluating rules. This is used for basic "smoke testing" of the rules engine. + * Multicast packets and packets destined for bridged devices treated a little differently. They are matched more than once. They are matched at the point of send with a NULL ZeroTier destination address, meaning that `MATCH_DEST_ZEROTIER_ADDRESS` is useless. That's because the true VL1 destination is not yet known. Then they are matched again for each true VL1 destination. On these later subsequent matches TEE actions are ignored and REDIRECT rules are interpreted as DROPs. This prevents multiple TEE or REDIRECT packets from being sent to third party devices. + * Rules in capabilities are always matched as if the current device is the sender (inbound == false). A capability specifies sender side rules that can be enforced on both sides. #### `/controller/network//member` @@ -228,6 +249,7 @@ Note that managed IP assignments are only used if they fall within a managed rou | clientMajorVersion | integer | Client major version or -1 if unknown | | clientMinorVersion | integer | Client minor version or -1 if unknown | | clientRevision | integer | Client revision or -1 if unknown | +| clientProtocolVersion | integer | ZeroTier protocol version reported by client | | fromAddr | string | Physical address if known | The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index a3d82adb..9dafcaa6 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -101,6 +101,11 @@ extern "C" { */ #define ZT_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS 4096 +/** + * Rules engine revision ID, which specifies rules engine capabilities + */ +#define ZT_RULES_ENGINE_REVISION 1 + /** * Maximum number of base (non-capability) network rules */ @@ -114,7 +119,7 @@ extern "C" { /** * Maximum number of per-member tags per network */ -#define ZT_MAX_NETWORK_TAGS 16 +#define ZT_MAX_NETWORK_TAGS 128 /** * Maximum number of direct network paths to a given peer diff --git a/node/Network.cpp b/node/Network.cpp index 1267f99c..0bbf070c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -38,36 +38,6 @@ // Uncomment to enable ZT_NETWORK_RULE_ACTION_DEBUG_LOG rule output to STDOUT #define ZT_RULES_ENGINE_DEBUGGING 1 -/* -{ - "name": "filter_log_test", - "private": true, - "v4AssignMode": { - "zt": true - }, - "v6AssignMode": { - "rfc4193": true, - "zt": false, - "6plane": false - }, - "routes": [ - { "target": "10.140.140.0/24", "via": null } - ], - "ipAssignmentPools": [ - { "ipRangeStart": "10.140.140.2", "ipRangeEnd": "10.140.140.254" } - ], - "rules": [ - { "type": "MATCH_ETHERTYPE", "etherType": 0x0800 }, - { "type": "ACTION_DEBUG_LOG" }, - - { "type": "MATCH_ETHERTYPE", "etherType": 0x0800, "not": true }, - { "type": "ACTION_DEBUG_LOG" }, - - { "type": "ACTION_ACCEPT" } - ] -} -*/ - namespace ZeroTier { #ifdef ZT_RULES_ENGINE_DEBUGGING @@ -162,7 +132,7 @@ static int _doZtFilter( #ifdef ZT_RULES_ENGINE_DEBUGGING std::vector dlog; char dpbuf[1024]; -#endif +#endif // ZT_RULES_ENGINE_DEBUGGING for(unsigned int rn=0;rn%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + printf(" _ " ZT_EOL_S); + for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) + printf(" | %s" ZT_EOL_S,m->c_str()); + printf(" + MATCH %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, ztSource.toString().c_str(), ztDest.toString().c_str(), (unsigned int)macSource[0], @@ -225,10 +207,8 @@ static int _doZtFilter( frameLen, etherType ); - for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) - printf(" %s" ZT_EOL_S,m->c_str()); - dlog.clear(); } + dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation continue; @@ -793,6 +773,7 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES,(uint64_t)ZT_MAX_CAPABILITY_RULES); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 22ffb1cf..67126d64 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -107,6 +107,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv" #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv" +// Rules engine revision +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV "revr" // Maximum number of rules per network this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES "mr" // Maximum number of capabilities this node can accept diff --git a/one.cpp b/one.cpp index 3cb6b775..7cddb376 100644 --- a/one.cpp +++ b/one.cpp @@ -1086,6 +1086,12 @@ int main(int argc,char **argv) } } + // This can be removed once the new controller code has been around for many versions + if (OSUtils::fileExists((homeDir + ZT_PATH_SEPARATOR_S + "controller.db").c_str(),true)) { + fprintf(stderr,"%s: FATAL: an old controller.db exists in %s -- see instructions in controller/README.md for how to migrate!" ZT_EOL_S,argv[0],homeDir.c_str()); + return 1; + } + #ifdef __UNIX_LIKE__ #ifndef ZT_ONE_NO_ROOT_CHECK if ((!skipRootCheck)&&(getuid() != 0)) { -- cgit v1.2.3 From 90f3e94565c735858af3e8ad891eab48f4c09659 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 12:21:44 -0700 Subject: Always output trace info when debugging rules. --- node/Network.cpp | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 0bbf070c..629e37d9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -183,31 +183,30 @@ static int _doZtFilter( } continue; case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: #ifdef ZT_RULES_ENGINE_DEBUGGING - if (thisSetMatches) { - printf(" _ " ZT_EOL_S); - for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) - printf(" | %s" ZT_EOL_S,m->c_str()); - printf(" + MATCH %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, - ztSource.toString().c_str(), - ztDest.toString().c_str(), - (unsigned int)macSource[0], - (unsigned int)macSource[1], - (unsigned int)macSource[2], - (unsigned int)macSource[3], - (unsigned int)macSource[4], - (unsigned int)macSource[5], - (unsigned int)macDest[0], - (unsigned int)macDest[1], - (unsigned int)macDest[2], - (unsigned int)macDest[3], - (unsigned int)macDest[4], - (unsigned int)macDest[5], - (int)inbound, - (int)noRedirect, - frameLen, - etherType - ); - } + printf(" _ " ZT_EOL_S); + for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) + printf(" | %s" ZT_EOL_S,m->c_str()); + printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + ((thisSetMatches) ? 'Y' : 'n'), + ztSource.toString().c_str(), + ztDest.toString().c_str(), + (unsigned int)macSource[0], + (unsigned int)macSource[1], + (unsigned int)macSource[2], + (unsigned int)macSource[3], + (unsigned int)macSource[4], + (unsigned int)macSource[5], + (unsigned int)macDest[0], + (unsigned int)macDest[1], + (unsigned int)macDest[2], + (unsigned int)macDest[3], + (unsigned int)macDest[4], + (unsigned int)macDest[5], + (int)inbound, + (int)noRedirect, + frameLen, + etherType + ); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation -- cgit v1.2.3 From fb5217761b4b2e294902646733fbff25436a1781 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 13:20:55 -0700 Subject: Add missing names in filter debug code. --- node/Network.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 629e37d9..047ecdc5 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -67,6 +67,11 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: return "MATCH_FRAME_SIZE_RANGE"; + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: return "MATCH_TAGS_SAMENESS"; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: return "MATCH_TAGS_BITWISE_AND"; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: return "MATCH_TAGS_BITWISE_OR"; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: return "MATCH_TAGS_BITWISE_XOR"; default: return "BAD_RULE_TYPE"; } } -- cgit v1.2.3 From 6bd5aba4faeced8879ce095d138279a84fce497a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 13:26:26 -0700 Subject: fix frame size range bug --- node/Capability.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index ce2eedc4..46fa9da6 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -323,7 +323,7 @@ public: break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: rules[ruleCount].v.frameSize[0] = b.template at(p); - rules[ruleCount].v.frameSize[0] = b.template at(p + 2); + rules[ruleCount].v.frameSize[1] = b.template at(p + 2); break; case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: -- cgit v1.2.3 From a3c7627acffc1f425477d9cd5c740980b31e5f1e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 14:43:16 -0700 Subject: Push more than one packet for credentials if we happen to have a whole lot. Should not happen often but might if a member has tons of tags. --- node/Membership.cpp | 86 ++++++++++++++++++++++++++++------------------------- node/Membership.hpp | 3 +- 2 files changed, 46 insertions(+), 43 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index 7ced9dcf..e809e2bd 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -28,61 +28,65 @@ namespace ZeroTier { -bool Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) +void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) { if ((now - _lastPushAttempt) < 1000ULL) - return false; + return; _lastPushAttempt = now; try { - Buffer capsAndTags; + bool unfinished; + do { + unfinished = false; + Buffer capsAndTags; - unsigned int appendedCaps = 0; - if (cap) { - capsAndTags.addSize(2); - std::map::iterator cs(_caps.find(cap->id())); - if ((cs != _caps.end())&&((now - cs->second.lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY)) { - cap->serialize(capsAndTags); - cs->second.lastPushed = now; - ++appendedCaps; + unsigned int appendedCaps = 0; + if (cap) { + capsAndTags.addSize(2); + std::map::iterator cs(_caps.find(cap->id())); + if ((cs != _caps.end())&&((now - cs->second.lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY)) { + cap->serialize(capsAndTags); + cs->second.lastPushed = now; + ++appendedCaps; + } + capsAndTags.setAt(0,(uint16_t)appendedCaps); + } else { + capsAndTags.append((uint16_t)0); } - capsAndTags.setAt(0,(uint16_t)appendedCaps); - } else { - capsAndTags.append((uint16_t)0); - } - unsigned int appendedTags = 0; - const unsigned int tagCountPos = capsAndTags.size(); - capsAndTags.addSize(2); - for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { - if ((capsAndTags.size() + sizeof(Tag)) > (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) - break; - nconf.tags[i].serialize(capsAndTags); - ts->lastPushed = now; - ++appendedTags; + unsigned int appendedTags = 0; + const unsigned int tagCountPos = capsAndTags.size(); + capsAndTags.addSize(2); + for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { + if ((capsAndTags.size() + sizeof(Tag)) >= (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) { + unfinished = true; + break; + } + nconf.tags[i].serialize(capsAndTags); + ts->lastPushed = now; + ++appendedTags; + } } - } - capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); + capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); - if ( (needCom) || (appendedCaps) || (appendedTags) ) { - Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - if (needCom) { - nconf.com.serialize(outp); - _lastPushedCom = now; + const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); + if ( (needCom) || (appendedCaps) || (appendedTags) ) { + Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + if (needCom) { + nconf.com.serialize(outp); + _lastPushedCom = now; + } + outp.append((uint8_t)0x00); + outp.append(capsAndTags.data(),capsAndTags.size()); + outp.compress(); + RR->sw->send(outp,true); } - outp.append((uint8_t)0x00); - outp.append(capsAndTags.data(),capsAndTags.size()); - outp.compress(); - RR->sw->send(outp,true); - return true; - } + } while (unfinished); // if there are many tags, etc., we can send more than one } catch ( ... ) { TRACE("unable to send credentials due to unexpected exception"); } - return false; } int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) diff --git a/node/Membership.hpp b/node/Membership.hpp index 580aa529..90363e2e 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -127,9 +127,8 @@ public: * @param peerAddress Address of member peer * @param nconf My network config * @param cap Capability to send or 0 if none - * @return True if we pushed something */ - bool sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); + void sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); /** * @param nconf Our network config -- cgit v1.2.3 From a5383d83d8ab8ce23cb1d12f8add0a3529c85506 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 15:25:00 -0700 Subject: Do not TEE or REDIRECT to self. --- node/Network.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 047ecdc5..4081d4e9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -165,8 +165,9 @@ static int _doZtFilter( continue; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { - if (!noRedirect) { - Packet outp(Address(rules[rn].v.fwd.address),RR->identity.address(),Packet::VERB_EXT_FRAME); + const Address fwdAddr(rules[rn].v.fwd.address); + if ((!noRedirect)&&(fwdAddr != RR->identity.address())) { + Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(nconf.networkId); outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); macDest.appendTo(outp); -- cgit v1.2.3 From e7dff1c785575372d84ad1923b6605bc8a3b7223 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 15:28:31 -0700 Subject: Change logic a little for self-as-destination in TEE and REDIRECT. --- node/Network.cpp | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 4081d4e9..24b1917c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -166,25 +166,35 @@ static int _doZtFilter( case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { const Address fwdAddr(rules[rn].v.fwd.address); - if ((!noRedirect)&&(fwdAddr != RR->identity.address())) { - Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(nconf.networkId); - outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); - outp.compress(); - RR->sw->send(outp,true); - } - - if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { - return -1; // match, drop packet (we redirected it) - } else { + if (fwdAddr == RR->identity.address()) { + // If we are the TEE or REDIRECT destination, don't TEE or REDIRECT + // to self. We should also accept here instead of interpreting + // REDIRECT as DROP since we are the destination. #ifdef ZT_RULES_ENGINE_DEBUGGING dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // TEE does not terminate evaluation + } else { + if (!noRedirect) { + Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(nconf.networkId); + outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); + outp.compress(); + RR->sw->send(outp,true); + } + + if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { + return -1; // match, drop packet (we redirected it) + } else { +#ifdef ZT_RULES_ENGINE_DEBUGGING + dlog.clear(); +#endif // ZT_RULES_ENGINE_DEBUGGING + thisSetMatches = 1; // TEE does not terminate evaluation + } } } continue; case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: -- cgit v1.2.3 From 7223685b9665590f2b0872060dd762706d371c4e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 26 Aug 2016 15:30:20 -0700 Subject: . --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 24b1917c..ecef2fa6 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -173,7 +173,7 @@ static int _doZtFilter( #ifdef ZT_RULES_ENGINE_DEBUGGING dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // TEE does not terminate evaluation + thisSetMatches = 1; } else { if (!noRedirect) { Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); -- cgit v1.2.3 From 51a420671f123968e416f5cce326e8e47aa71643 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 15:17:34 -0700 Subject: Make rules engine debug a bit more verbose. --- node/Network.cpp | 76 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 26 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index ecef2fa6..5fd7ac8c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -75,6 +75,38 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) default: return "BAD_RULE_TYPE"; } } +static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool noRedirect,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) +{ + printf("!! %c %s inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + ((thisSetMatches) ? 'Y' : '.'), + ruleName, + (int)inbound, + (int)noRedirect, + frameLen, + etherType + ); + for(std::vector::const_iterator m(dlog.begin());m!=dlog.end();++m) + printf(" | %s" ZT_EOL_S,m->c_str()); + printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" ZT_EOL_S, + ((thisSetMatches) ? 'Y' : '.'), + ztSource.toString().c_str(), + ztDest.toString().c_str(), + (unsigned int)macSource[0], + (unsigned int)macSource[1], + (unsigned int)macSource[2], + (unsigned int)macSource[3], + (unsigned int)macSource[4], + (unsigned int)macSource[5], + (unsigned int)macDest[0], + (unsigned int)macDest[1], + (unsigned int)macDest[2], + (unsigned int)macDest[3], + (unsigned int)macDest[4], + (unsigned int)macDest[5] + ); + if (msg) + printf(" + (%s)" ZT_EOL_S,msg); +} #else #define FILTER_TRACE(f,...) {} #endif // ZT_RULES_ENGINE_DEBUGGING @@ -145,9 +177,13 @@ static int _doZtFilter( switch(rt) { case ZT_NETWORK_RULE_ACTION_DROP: if (thisSetMatches) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace("ACTION_DROP",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); +#endif // ZT_RULES_ENGINE_DEBUGGING return -1; // match, drop packet } else { #ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace("ACTION_DROP",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // no match, evaluate next set @@ -155,9 +191,13 @@ static int _doZtFilter( continue; case ZT_NETWORK_RULE_ACTION_ACCEPT: if (thisSetMatches) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); +#endif // ZT_RULES_ENGINE_DEBUGGING return 1; // match, accept packet } else { #ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // no match, evaluate next set @@ -171,6 +211,7 @@ static int _doZtFilter( // to self. We should also accept here instead of interpreting // REDIRECT as DROP since we are the destination. #ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"ignored since we are the destination"); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; @@ -188,41 +229,22 @@ static int _doZtFilter( } if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(noRedirect) ? "second-pass match, not actually redirecting" : (const char *)0); +#endif // ZT_RULES_ENGINE_DEBUGGING return -1; // match, drop packet (we redirected it) } else { #ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace("ACTION_TEE",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(noRedirect) ? "second-pass match, not actually teeing" : (const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // TEE does not terminate evaluation } } } continue; - case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: // a no-op target specifically for debugging purposes #ifdef ZT_RULES_ENGINE_DEBUGGING - printf(" _ " ZT_EOL_S); - for(std::vector::iterator m(dlog.begin());m!=dlog.end();++m) - printf(" | %s" ZT_EOL_S,m->c_str()); - printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, - ((thisSetMatches) ? 'Y' : 'n'), - ztSource.toString().c_str(), - ztDest.toString().c_str(), - (unsigned int)macSource[0], - (unsigned int)macSource[1], - (unsigned int)macSource[2], - (unsigned int)macSource[3], - (unsigned int)macSource[4], - (unsigned int)macSource[5], - (unsigned int)macDest[0], - (unsigned int)macDest[1], - (unsigned int)macDest[2], - (unsigned int)macDest[3], - (unsigned int)macDest[4], - (unsigned int)macDest[5], - (int)inbound, - (int)noRedirect, - frameLen, - etherType - ); + _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation @@ -458,7 +480,9 @@ static int _doZtFilter( } } break; - default: continue; + default: // rules we don't know do not match -- this means upgrading may be necessary before shipping new rules on a network or old clients might get blocked + thisRuleMatches = 0; + break; } // thisSetMatches remains true if the current rule matched (or did NOT match if NOT bit is set) -- cgit v1.2.3 From f0636ffd4a86336f00bbf77a3e303def6261b518 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 15:54:06 -0700 Subject: EXT_FRAME messages should always be accepted if we are the destination for a matching TEE or REDIRECT rule. --- node/IncomingPacket.cpp | 46 +++++++++++++++++++++++----------------------- node/Network.cpp | 35 ++++++++++++++++++----------------- node/Network.hpp | 4 ++-- 3 files changed, 43 insertions(+), 42 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 0804f04a..c75125d3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -596,13 +596,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE); - const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO); const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM); + const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); + const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); @@ -610,27 +608,29 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

address(),network->id())) { - if (network->config().permitsBridging(peer->address())) { - network->learnBridgeRoute(from,peer->address()); - } else { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay - return true; - } - } else if (to != network->mac()) { - if (!network->config().permitsBridging(RR->identity.address())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay - return true; - } + switch (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { + case 1: + if (from != MAC(peer->address(),network->id())) { + if (network->config().permitsBridging(peer->address())) { + network->learnBridgeRoute(from,peer->address()); + } else { + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + return true; + } + } else if (to != network->mac()) { + if (!network->config().permitsBridging(RR->identity.address())) { + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + return true; + } + } + // fall through -- 2 means accept regardless of bridging state + case 2: + RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); + break; } - const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); - const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); - if (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) - RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } } else { diff --git a/node/Network.cpp b/node/Network.cpp index 5fd7ac8c..fa8cad80 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -77,7 +77,9 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) } static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool noRedirect,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) { - printf("!! %c %s inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + static volatile unsigned long cnt = 0; + printf("%.6lu %c %s inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + cnt, ((thisSetMatches) ? 'Y' : '.'), ruleName, (int)inbound, @@ -86,8 +88,8 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b etherType ); for(std::vector::const_iterator m(dlog.begin());m!=dlog.end();++m) - printf(" | %s" ZT_EOL_S,m->c_str()); - printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" ZT_EOL_S, + printf(" | %s" ZT_EOL_S,m->c_str()); + printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" ZT_EOL_S, ((thisSetMatches) ? 'Y' : '.'), ztSource.toString().c_str(), ztDest.toString().c_str(), @@ -105,7 +107,7 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b (unsigned int)macDest[5] ); if (msg) - printf(" + (%s)" ZT_EOL_S,msg); + printf(" + (%s)" ZT_EOL_S,msg); } #else #define FILTER_TRACE(f,...) {} @@ -140,7 +142,7 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } -// 0 == no match, -1 == match/drop, 1 == match/accept +// 0 == no match, -1 == match/drop, 1 == match/accept, 2 == match/accept even if bridged static int _doZtFilter( const RuntimeEnvironment *RR, const bool noRedirect, @@ -212,9 +214,8 @@ static int _doZtFilter( // REDIRECT as DROP since we are the destination. #ifdef ZT_RULES_ENGINE_DEBUGGING _dumpFilterTrace(_rtn(rt),thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"ignored since we are the destination"); - dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; + return 2; // we should "super-accept" this packet since we are the TEE or REDIRECT destination } else { if (!noRedirect) { Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); @@ -580,6 +581,7 @@ bool Network::filterOutgoingPacket( m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return false; case 1: + case 2: if (ztDest) m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return true; @@ -592,6 +594,7 @@ bool Network::filterOutgoingPacket( m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); return false; case 1: + case 2: if (ztDest) m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,&(_config.capabilities[c])); return true; @@ -601,7 +604,7 @@ bool Network::filterOutgoingPacket( return false; } -bool Network::filterIncomingPacket( +int Network::filterIncomingPacket( const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, @@ -620,24 +623,22 @@ bool Network::filterIncomingPacket( const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: - return false; - case 1: - return true; + case -1: return 0; + case 1: return 1; + case 2: return 2; } Membership::CapabilityIterator mci(m); const Capability *c; while ((c = mci.next(_config))) { switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: - return false; - case 1: - return true; + case -1: return 0; + case 1: return 1; + case 2: return 2; } } - return false; + return 0; } bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const diff --git a/node/Network.hpp b/node/Network.hpp index c5e7d570..aa4b67f8 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -124,9 +124,9 @@ public: * @param frameLen Ethernet frame payload length * @param etherType 16-bit ethernet type ID * @param vlanId 16-bit VLAN ID - * @return True if packet should be accepted locally + * @return 0 == drop, 1 == accept, 2 == accept even if bridged */ - bool filterIncomingPacket( + int filterIncomingPacket( const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, -- cgit v1.2.3 From cb8219333396a1ec72dc96445262440bc5e2b6ec Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 16:19:26 -0700 Subject: Debug output fixes. --- node/Network.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index fa8cad80..ac583b84 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -79,7 +79,7 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b { static volatile unsigned long cnt = 0; printf("%.6lu %c %s inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, - cnt, + cnt++, ((thisSetMatches) ? 'Y' : '.'), ruleName, (int)inbound, @@ -88,8 +88,8 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b etherType ); for(std::vector::const_iterator m(dlog.begin());m!=dlog.end();++m) - printf(" | %s" ZT_EOL_S,m->c_str()); - printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" ZT_EOL_S, + printf(" | %s" ZT_EOL_S,m->c_str()); + printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" ZT_EOL_S, ((thisSetMatches) ? 'Y' : '.'), ztSource.toString().c_str(), ztDest.toString().c_str(), @@ -213,7 +213,7 @@ static int _doZtFilter( // to self. We should also accept here instead of interpreting // REDIRECT as DROP since we are the destination. #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"ignored since we are the destination"); + _dumpFilterTrace(_rtn(rt),thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT resulted in 'super-accept' since we are destination"); #endif // ZT_RULES_ENGINE_DEBUGGING return 2; // we should "super-accept" this packet since we are the TEE or REDIRECT destination } else { -- cgit v1.2.3 From ac1c127b68c9c23f67f4940af60312e23d64f93d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 16:24:08 -0700 Subject: Debug output fixes. --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index ac583b84..f7969131 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -107,7 +107,7 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b (unsigned int)macDest[5] ); if (msg) - printf(" + (%s)" ZT_EOL_S,msg); + printf(" + (%s)" ZT_EOL_S,msg); } #else #define FILTER_TRACE(f,...) {} -- cgit v1.2.3 From cb63babac40d19b7495bdf3b03f5808612976783 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 29 Aug 2016 16:38:10 -0700 Subject: Debug output fixes. --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index f7969131..0279bc53 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -107,7 +107,7 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b (unsigned int)macDest[5] ); if (msg) - printf(" + (%s)" ZT_EOL_S,msg); + printf(" + (%s)" ZT_EOL_S,msg); } #else #define FILTER_TRACE(f,...) {} -- cgit v1.2.3 From 2ff2a8fd9ad195c11b2738306392802a323fb854 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 09:38:21 -0700 Subject: Cluster build fixes and warning elimination. --- node/Cluster.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index f590ad1c..933bfb37 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -455,7 +455,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) Mutex::Lock _l2(_members[fromMemberId].lock); _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size()); } - RR->sw->send(rendezvousForLocal,true,0); + RR->sw->send(rendezvousForLocal,true); } } } break; @@ -466,7 +466,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) const unsigned int len = dmsg.at(ptr); ptr += 2; Packet outp(rcpt,RR->identity.address(),verb); outp.append(dmsg.field(ptr,len),len); ptr += len; - RR->sw->send(outp,true,0); + RR->sw->send(outp,true); //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); } break; } @@ -719,7 +719,9 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr std::vector best; const double currentDistance = _dist3d(_x,_y,_z,px,py,pz); double bestDistance = (offload ? 2147483648.0 : currentDistance); +#ifdef ZT_TRACE unsigned int bestMember = _id; +#endif { Mutex::Lock _l(_memberIds_m); for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { @@ -731,7 +733,9 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz); if (mdist < bestDistance) { bestDistance = mdist; +#ifdef ZT_TRACE bestMember = *mid; +#endif best = m.zeroTierPhysicalEndpoints; } } -- cgit v1.2.3 From 8e3004591bae72e20232200b74fe145f5574d02e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 14:01:15 -0700 Subject: Add overlooked MATCH_ICMP to rule set. --- controller/EmbeddedNetworkController.cpp | 20 ++++++++++++ include/ZeroTierOne.h | 30 +++++++++++++----- node/Capability.hpp | 11 +++++++ node/Network.cpp | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5c52cfcb..0a1ee2ef 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -221,6 +221,14 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["not"] = ((rule.t & 0x80) != 0); r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; break; + case ZT_NETWORK_RULE_MATCH_ICMP: + r["type"] = "MATCH_ICMP"; + r["not"] = ((rule.t & 0x80) != 0); + r["type"] = (unsigned int)rule.v.icmp.type; + if ((rule.v.icmp.flags & 0x01) != 0) + r["code"] = (unsigned int)rule.v.icmp.code; + else r["code"] = json(); + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; r["not"] = ((rule.t & 0x80) != 0); @@ -375,6 +383,18 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); return true; + } else if (t == "MATCH_ICMP") { + rule.t |= ZT_NETWORK_RULE_MATCH_ICMP; + rule.v.icmp.type = (uint8_t)(_jI(r["type"],0ULL) & 0xffULL); + json &code = r["code"]; + if (code.is_null()) { + rule.v.icmp.code = 0; + rule.v.icmp.flags = 0x00; + } else { + rule.v.icmp.code = (uint8_t)(_jI(code,0ULL) & 0xffULL); + rule.v.icmp.flags = 0x01; + } + return true; } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; rule.v.port[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 9dafcaa6..121de7a2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -597,45 +597,50 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_MATCH_IP_PROTOCOL = 45, + /** + * ICMP type and possibly code (does not match if not ICMP) + */ + ZT_NETWORK_RULE_MATCH_ICMP = 46, + /** * IP source port range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 46, + ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 47, /** * IP destination port range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 47, + ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 48, /** * Packet characteristics (set of flags) */ - ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 48, + ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 49, /** * Frame size range (start-end, inclusive) */ - ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 49, + ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 50, /** * Match if local and remote tags differ by no more than value, use 0 to check for equality */ - ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 50, + ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 51, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 51, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 52, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 52, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 53, /** * Match if local and remote tags XORed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 53 + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 54 }; /** @@ -739,6 +744,15 @@ typedef struct */ uint16_t frameSize[2]; + /** + * ICMP type and code + */ + struct { + uint8_t type; // ICMP type, always matched + uint8_t code; // ICMP code if matched + uint8_t flags; // flag 0x01 means also match code, otherwise only match type + } icmp; + /** * For tag-related rules */ diff --git a/node/Capability.hpp b/node/Capability.hpp index 46fa9da6..f62ed30b 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -233,6 +233,12 @@ public: b.append((uint8_t)1); b.append((uint8_t)rules[i].v.ipProtocol); break; + case ZT_NETWORK_RULE_MATCH_ICMP: + b.append((uint8_t)3); + b.append((uint8_t)rules[i].v.icmp.type); + b.append((uint8_t)rules[i].v.icmp.code); + b.append((uint8_t)rules[i].v.icmp.flags); + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: b.append((uint8_t)4); @@ -312,6 +318,11 @@ public: case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; break; + case ZT_NETWORK_RULE_MATCH_ICMP: + rules[ruleCount].v.icmp.type = (uint8_t)b[p]; + rules[ruleCount].v.icmp.code = (uint8_t)b[p+1]; + rules[ruleCount].v.icmp.flags = (uint8_t)b[p+2]; + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: rules[ruleCount].v.port[0] = b.template at(p); diff --git a/node/Network.cpp b/node/Network.cpp index 0279bc53..026a07fc 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -64,6 +64,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IPV6_DEST: return "MATCH_IPV6_DEST"; case ZT_NETWORK_RULE_MATCH_IP_TOS: return "MATCH_IP_TOS"; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: return "MATCH_IP_PROTOCOL"; + case ZT_NETWORK_RULE_MATCH_ICMP: return "MATCH_ICMP"; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; @@ -362,6 +363,57 @@ static int _doZtFilter( FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; + case ZT_NETWORK_RULE_MATCH_ICMP: + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + if (frameData[9] == 0x01) { + const unsigned int ihl = (frameData[0] & 0xf) * 32; + if (frameLen >= (ihl + 2)) { + if (rules[rn].v.icmp.type == frameData[ihl]) { + if ((rules[rn].v.icmp.flags & 0x01) != 0) { + thisRuleMatches = (uint8_t)(frameData[ihl+1] == rules[rn].v.icmp.code); + } else { + thisRuleMatches = 1; + } + } else { + thisRuleMatches = 0; + } + FILTER_TRACE("%u %s %c (IPv4) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[ihl],(int)rules[rn].v.icmp.type,(int)frameData[ihl+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [IPv4 frame invalid] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not ICMP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else if (etherType == ZT_ETHERTYPE_IPV6) { + unsigned int pos = 0,proto = 0; + if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { + if ((proto == 0x3a)&&(frameLen >= (pos+2))) { + if (rules[rn].v.icmp.type == frameData[pos]) { + if ((rules[rn].v.icmp.flags & 0x01) != 0) { + thisRuleMatches = (uint8_t)(frameData[pos+1] == rules[rn].v.icmp.code); + } else { + thisRuleMatches = 1; + } + } else { + thisRuleMatches = 0; + } + FILTER_TRACE("%u %s %c (IPv4) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[pos],(int)rules[rn].v.icmp.type,(int)frameData[pos+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not ICMPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } + break; + break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { -- cgit v1.2.3 From 54489a7f61a19b07eaa5a87d1df2ee30101f29ee Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 14:14:58 -0700 Subject: rename SAMENESS to DIFFERENCE which is less confusing --- controller/EmbeddedNetworkController.cpp | 8 +++---- include/ZeroTierOne.h | 2 +- node/Capability.hpp | 21 ++++++------------ node/Network.cpp | 37 ++++++++++++++++++-------------- 4 files changed, 33 insertions(+), 35 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0a1ee2ef..29dd8ad7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -255,8 +255,8 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["start"] = (unsigned int)rule.v.frameSize[0]; r["end"] = (unsigned int)rule.v.frameSize[1]; break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: - r["type"] = "MATCH_TAGS_SAMENESS"; + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: + r["type"] = "MATCH_TAGS_DIFFERENCE"; r["not"] = ((rule.t & 0x80) != 0); r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; @@ -431,8 +431,8 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; - } else if (t == "MATCH_TAGS_SAMENESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; + } else if (t == "MATCH_TAGS_DIFFERENCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 121de7a2..73450006 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -625,7 +625,7 @@ enum ZT_VirtualNetworkRuleType /** * Match if local and remote tags differ by no more than value, use 0 to check for equality */ - ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS = 51, + ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 51, /** * Match if local and remote tags ANDed together equal value. diff --git a/node/Capability.hpp b/node/Capability.hpp index f62ed30b..8e749e80 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -38,9 +38,6 @@ class RuntimeEnvironment; /** * A set of grouped and signed network flow rules * - * The use of capabilities implements capability-based security on ZeroTIer - * virtual networks for efficient and manageable network micro-segmentation. - * * On the sending side the sender does the following for each packet: * * (1) Evaluates its capabilities in ascending order of ID to determine @@ -49,16 +46,12 @@ class RuntimeEnvironment; * receving peer ("presents" it). * (3) The sender then sends the packet. * - * On the receiving side the receiver does the following for each packet: - * - * (1) Evaluates the capabilities of the sender (that the sender has - * presented) to determine if it should received this packet. - * (2) Evaluates its own capabilities to determine if it should receive - * this packet. - * (3) If both check out, it receives the packet. + * On the receiving side the receiver evaluates the capabilities presented + * by the sender. If any valid un-expired capability allows this packet it + * is accepted. * - * Note that rules in capabilities can do other things as well such as TEE - * or REDIRECT packets. See filter code and ZT_VirtualNetworkRule. + * Note that this is after evaluation of network scope rules and only if + * network scope rules do not deliver an explicit match. */ class Capability { @@ -255,7 +248,7 @@ public: b.append((uint16_t)rules[i].v.frameSize[0]); b.append((uint16_t)rules[i].v.frameSize[1]); break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: @@ -336,7 +329,7 @@ public: rules[ruleCount].v.frameSize[0] = b.template at(p); rules[ruleCount].v.frameSize[1] = b.template at(p + 2); break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: diff --git a/node/Network.cpp b/node/Network.cpp index 026a07fc..f8b7c1d5 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -69,7 +69,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: return "MATCH_FRAME_SIZE_RANGE"; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: return "MATCH_TAGS_SAMENESS"; + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: return "MATCH_TAGS_DIFFERENCE"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: return "MATCH_TAGS_BITWISE_AND"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: return "MATCH_TAGS_BITWISE_OR"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: return "MATCH_TAGS_BITWISE_XOR"; @@ -487,7 +487,7 @@ static int _doZtFilter( thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { @@ -510,13 +510,18 @@ static int _doZtFilter( } } if (!rtv) { - thisRuleMatches = 0; - FILTER_TRACE("%u %s %c remote tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + if (inbound) { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } else { + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } } else { - if (rt == ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS) { - const uint32_t sameness = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); - thisRuleMatches = (uint8_t)(sameness <= rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u sameness:%u <= %u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,sameness,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); + if (rt == ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE) { + const uint32_t diff = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); + thisRuleMatches = (uint8_t)(diff <= rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { thisRuleMatches = (uint8_t)((lt->value() & *rtv) == rules[rn].v.tag.value); FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); @@ -675,22 +680,22 @@ int Network::filterIncomingPacket( const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; - case 1: return 1; - case 2: return 2; + case -1: return 0; // DROP + case 1: return 1; // ACCEPT + case 2: return 2; // super-ACCEPT } Membership::CapabilityIterator mci(m); const Capability *c; while ((c = mci.next(_config))) { - switch(_doZtFilter(RR,false,_config,false,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; - case 1: return 1; - case 2: return 2; + switch(_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { + case -1: return 0; // DROP + case 1: return 1; // ACCEPT + case 2: return 2; // super-ACCEPT } } - return 0; + return 0; // DROP } bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const -- cgit v1.2.3 From 74afef8eb1d96aec291c6dfeca31e69a2ad33d69 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 16:50:22 -0700 Subject: Think through and refine a few things in rules, especially edge case TEE and REDIRECT behavior and semantics. --- controller/EmbeddedNetworkController.cpp | 2 - node/Capability.hpp | 2 +- node/IncomingPacket.cpp | 6 +- node/Network.cpp | 290 +++++++++++++++++++++++-------- node/Network.hpp | 11 +- node/OutboundMulticast.cpp | 5 +- node/Packet.hpp | 5 +- node/Switch.cpp | 5 +- 8 files changed, 229 insertions(+), 97 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 29dd8ad7..ff2f34ec 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -144,7 +144,6 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["type"] = "ACTION_REDIRECT"; r["address"] = Address(rule.v.fwd.address).toString(); r["flags"] = (unsigned int)rule.v.fwd.flags; - r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: r["type"] = "ACTION_DEBUG_LOG"; @@ -308,7 +307,6 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); - rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_DEBUG_LOG") { rule.t |= ZT_NETWORK_RULE_ACTION_DEBUG_LOG; diff --git a/node/Capability.hpp b/node/Capability.hpp index 8e749e80..e23d7943 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -178,7 +178,7 @@ public: b.append((uint8_t)14); b.append((uint64_t)rules[i].v.fwd.address); b.append((uint32_t)rules[i].v.fwd.flags); - b.append((uint16_t)rules[i].v.fwd.length); + b.append((uint16_t)rules[i].v.fwd.length); // unused for redirect break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c75125d3..4b013078 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -560,7 +560,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr const MAC sourceMac(peer->address(),network->id()); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0)) + if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true); } @@ -625,7 +625,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; @@ -981,7 +981,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0)) { + if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { RR->node->putFrame(network->id(),network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); } } diff --git a/node/Network.cpp b/node/Network.cpp index f8b7c1d5..5a9b07cf 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -76,15 +76,14 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) default: return "BAD_RULE_TYPE"; } } -static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool noRedirect,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) +static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) { static volatile unsigned long cnt = 0; - printf("%.6lu %c %s inbound=%d noRedirect=%d frameLen=%u etherType=%u" ZT_EOL_S, + printf("%.6lu %c %s %s frameLen=%u etherType=%u" ZT_EOL_S, cnt++, ((thisSetMatches) ? 'Y' : '.'), ruleName, - (int)inbound, - (int)noRedirect, + ((inbound) ? "INBOUND" : "OUTBOUND"), frameLen, etherType ); @@ -143,14 +142,20 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig return false; // overflow == invalid } -// 0 == no match, -1 == match/drop, 1 == match/accept, 2 == match/accept even if bridged -static int _doZtFilter( +enum _doZtFilterResult +{ + DOZTFILTER_NO_MATCH = 0, + DOZTFILTER_DROP = 1, + DOZTFILTER_REDIRECT = 2, + DOZTFILTER_ACCEPT = 3, + DOZTFILTER_SUPER_ACCEPT = 4 +}; +static _doZtFilterResult _doZtFilter( const RuntimeEnvironment *RR, - const bool noRedirect, const NetworkConfig &nconf, const bool inbound, const Address &ztSource, - const Address &ztDest, + Address &ztDest, // MUTABLE const MAC &macSource, const MAC &macDest, const uint8_t *const frameData, @@ -163,7 +168,9 @@ static int _doZtFilter( const unsigned int localTagCount, const uint32_t *const remoteTagIds, const uint32_t *const remoteTagValues, - const unsigned int remoteTagCount) + const unsigned int remoteTagCount, + Address &cc, // MUTABLE + unsigned int &ccLength) // MUTABLE { // For each set of rules we start by assuming that they match (since no constraints // yields a 'match all' rule). @@ -181,75 +188,83 @@ static int _doZtFilter( case ZT_NETWORK_RULE_ACTION_DROP: if (thisSetMatches) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DROP",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DROP",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return -1; // match, drop packet + return DOZTFILTER_DROP; } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DROP",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DROP",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // no match, evaluate next set + thisSetMatches = 1; } continue; + case ZT_NETWORK_RULE_ACTION_ACCEPT: if (thisSetMatches) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return 1; // match, accept packet + return DOZTFILTER_ACCEPT; // match, accept packet } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // no match, evaluate next set + thisSetMatches = 1; } continue; + case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { const Address fwdAddr(rules[rn].v.fwd.address); - if (fwdAddr == RR->identity.address()) { - // If we are the TEE or REDIRECT destination, don't TEE or REDIRECT - // to self. We should also accept here instead of interpreting - // REDIRECT as DROP since we are the destination. + if (fwdAddr == ztSource) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT resulted in 'super-accept' since we are destination"); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored since source is target"); #endif // ZT_RULES_ENGINE_DEBUGGING - return 2; // we should "super-accept" this packet since we are the TEE or REDIRECT destination - } else { - if (!noRedirect) { - Packet outp(fwdAddr,RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(nconf.networkId); - outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); - outp.compress(); - RR->sw->send(outp,true); + thisSetMatches = 1; + } else if (fwdAddr == RR->identity.address()) { + if (inbound) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT interpreted as super-accept since we are target"); +#endif // ZT_RULES_ENGINE_DEBUGGING + return DOZTFILTER_SUPER_ACCEPT; + } else { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored on outbound since we are target"); +#endif // ZT_RULES_ENGINE_DEBUGGING + thisSetMatches = 1; } - + } else if (fwdAddr == ztDest) { +#ifdef ZT_RULES_ENGINE_DEBUGGING + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored since destination is target"); +#endif // ZT_RULES_ENGINE_DEBUGGING + thisSetMatches = 1; + } else { if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(noRedirect) ? "second-pass match, not actually redirecting" : (const char *)0); + _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return -1; // match, drop packet (we redirected it) + ztDest = fwdAddr; + return DOZTFILTER_REDIRECT; } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_TEE",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(noRedirect) ? "second-pass match, not actually teeing" : (const char *)0); + _dumpFilterTrace("ACTION_TEE",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // TEE does not terminate evaluation + cc = fwdAddr; + ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; + thisSetMatches = 1; } } } continue; + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: // a no-op target specifically for debugging purposes #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,noRedirect,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; // DEBUG_LOG does not terminate evaluation + thisSetMatches = 1; continue; default: break; @@ -547,7 +562,7 @@ static int _doZtFilter( thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } - return 0; + return DOZTFILTER_NO_MATCH; } const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); @@ -614,7 +629,7 @@ Network::~Network() } bool Network::filterOutgoingPacket( - const bool noRedirect, + const bool noTee, const Address &ztSource, const Address &ztDest, const MAC &macSource, @@ -626,39 +641,94 @@ bool Network::filterOutgoingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + Address ztDest2(ztDest); + Address cc; + unsigned int ccLength = 0; + bool mainRuleTableMatch = false; + bool accept = false; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch(_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); + switch(_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + case DOZTFILTER_NO_MATCH: + break; + case DOZTFILTER_DROP: return false; - case 1: - case 2: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); - return true; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + mainRuleTableMatch = true; + accept = true; + break; } - for(unsigned int c=0;c<_config.capabilityCount;++c) { - switch (_doZtFilter(RR,noRedirect,_config,false,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,(const Capability *)0); - return false; - case 1: - case 2: - if (ztDest) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,&(_config.capabilities[c])); - return true; + const Capability *relevantCap = (const Capability *)0; + if (!mainRuleTableMatch) { + for(unsigned int c=0;c<_config.capabilityCount;++c) { + ztDest2 = ztDest; // sanity check + Address cc2; + unsigned int ccLength2 = 0; + switch (_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + case DOZTFILTER_NO_MATCH: + case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern + break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + if ((!noTee)&&(cc2)) { + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength2); + outp.compress(); + RR->sw->send(outp,true); + } + relevantCap = &(_config.capabilities[c]); + accept = true; + break; + } + if (accept) + break; } } - return false; + if (accept) { + if (ztDest2) + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); + + if ((!noTee)&&(cc)) { + Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength); + outp.compress(); + RR->sw->send(outp,true); + } + + if (ztDest != ztDest2) { + Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,frameLen); + outp.compress(); + RR->sw->send(outp,true); + return false; // DROP locally, since we redirected + } + } + + return accept; } int Network::filterIncomingPacket( @@ -673,29 +743,97 @@ int Network::filterIncomingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + Address ztDest2(ztDest); + Address cc; + unsigned int ccLength = 0; + bool mainRuleTableMatch = false; + int accept = 0; Mutex::Lock _l(_lock); Membership &m = _memberships[ztDest]; const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; // DROP - case 1: return 1; // ACCEPT - case 2: return 2; // super-ACCEPT + switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + case DOZTFILTER_NO_MATCH: + break; + case DOZTFILTER_DROP: + return 0; // DROP + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + mainRuleTableMatch = true; + accept = 1; // ACCEPT + break; + case DOZTFILTER_SUPER_ACCEPT: + mainRuleTableMatch = true; + accept = 2; // super-ACCEPT + break; } - Membership::CapabilityIterator mci(m); - const Capability *c; - while ((c = mci.next(_config))) { - switch(_doZtFilter(RR,false,_config,true,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount)) { - case -1: return 0; // DROP - case 1: return 1; // ACCEPT - case 2: return 2; // super-ACCEPT + if (!mainRuleTableMatch) { + Membership::CapabilityIterator mci(m); + const Capability *c; + while ((c = mci.next(_config))) { + ztDest2 = ztDest; // sanity check + Address cc2; + unsigned int ccLength2 = 0; + switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + case DOZTFILTER_NO_MATCH: + case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern + break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + accept = 1; // ACCEPT + break; + case DOZTFILTER_SUPER_ACCEPT: + accept = 2; // super-ACCEPT + break; + } + if (accept) { + if (cc2) { + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength2); + outp.compress(); + RR->sw->send(outp,true); + } + break; + } + } + } + + if (accept) { + if (cc) { + Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength); + outp.compress(); + RR->sw->send(outp,true); + } + + if (ztDest != ztDest2) { + Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,frameLen); + outp.compress(); + RR->sw->send(outp,true); + return 0; // DROP locally, since we redirected } } - return 0; // DROP + return accept; } bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const diff --git a/node/Network.hpp b/node/Network.hpp index aa4b67f8..45a51bf2 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -82,11 +82,10 @@ public: * Apply filters to an outgoing packet * * This applies filters from our network config and, if that doesn't match, - * our capabilities in ascending order of capability ID. If there is a match - * certain actions may be taken such as pushing credentials to ztDest and - * sending a copy of the packet to a TEE or REDIRECT target. + * our capabilities in ascending order of capability ID. Additional actions + * such as TEE may be taken, and credentials may be pushed. * - * @param noRedirect If true, do not TEE or REDIRECT -- this is set for secondary filtrations done in multicast and bridge send paths + * @param noTee If true, do not TEE anything anywhere * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -95,10 +94,10 @@ public: * @param frameLen Ethernet frame payload length * @param etherType 16-bit ethernet type ID * @param vlanId 16-bit VLAN ID - * @return True if packet should be sent to destination peer + * @return True if packet should be sent, false if dropped or redirected */ bool filterOutgoingPacket( - const bool noRedirect, + const bool noTee, const Address &ztSource, const Address &ztDest, const MAC &macSource, diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 6b583e7c..33c28f65 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -86,10 +86,11 @@ void OutboundMulticast::init( void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) { const SharedPtr nw(RR->node->network(_nwid)); - if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + Address toAddr2(toAddr); + if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); - _packet.setDestination(toAddr); + _packet.setDestination(toAddr2); RR->sw->send(_packet,true); } } diff --git a/node/Packet.hpp b/node/Packet.hpp index 570bace9..27e289fd 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -655,9 +655,8 @@ public: * * Flags: * 0x01 - Certificate of network membership attached (DEPRECATED) - * 0x02 - Packet is a TEE'd packet - * 0x04 - Packet is a REDIRECT'ed packet - * 0x08 - TEE/REDIRECT'ed packet is on inbound side of connection + * 0x02 - This is a TEE'd or REDIRECT'ed packet + * 0x04 - TEE/REDIRECT'ed packet is from inbound side * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we diff --git a/node/Switch.cpp b/node/Switch.cpp index 546c9157..8e41c89f 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -437,10 +437,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),len); - // We filter with a NULL destination ZeroTier address first. Filtrations - // for each ZT destination are also done in OutboundMulticast, but these - // set noRedirect to true. This prevents multiple TEEs and REDIRECTs for - // multicast packets. + // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; -- cgit v1.2.3 From 994b25af4e92a4a8b13fd1ce06fe47368d24508c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 17:45:55 -0700 Subject: Simplify some logic. --- node/Network.cpp | 147 +++++++++++++++++++++++++++---------------------------- 1 file changed, 71 insertions(+), 76 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 5a9b07cf..13566a80 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -643,8 +643,8 @@ bool Network::filterOutgoingPacket( uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; Address ztDest2(ztDest); Address cc; + const Capability *relevantCap = (const Capability *)0; unsigned int ccLength = 0; - bool mainRuleTableMatch = false; bool accept = false; Mutex::Lock _l(_lock); @@ -653,51 +653,49 @@ bool Network::filterOutgoingPacket( const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch(_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + case DOZTFILTER_NO_MATCH: + for(unsigned int c=0;c<_config.capabilityCount;++c) { + ztDest2 = ztDest; // sanity check + Address cc2; + unsigned int ccLength2 = 0; + switch (_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + case DOZTFILTER_NO_MATCH: + case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern + break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + if ((!noTee)&&(cc2)) { + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength2); + outp.compress(); + RR->sw->send(outp,true); + } + relevantCap = &(_config.capabilities[c]); + accept = true; + break; + } + if (accept) + break; + } break; + case DOZTFILTER_DROP: return false; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side - mainRuleTableMatch = true; accept = true; break; } - const Capability *relevantCap = (const Capability *)0; - if (!mainRuleTableMatch) { - for(unsigned int c=0;c<_config.capabilityCount;++c) { - ztDest2 = ztDest; // sanity check - Address cc2; - unsigned int ccLength2 = 0; - switch (_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { - case DOZTFILTER_NO_MATCH: - case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern - break; - case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() - case DOZTFILTER_ACCEPT: - case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side - if ((!noTee)&&(cc2)) { - Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,ccLength2); - outp.compress(); - RR->sw->send(outp,true); - } - relevantCap = &(_config.capabilities[c]); - accept = true; - break; - } - if (accept) - break; - } - } - if (accept) { if (ztDest2) m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); @@ -746,7 +744,6 @@ int Network::filterIncomingPacket( Address ztDest2(ztDest); Address cc; unsigned int ccLength = 0; - bool mainRuleTableMatch = false; int accept = 0; Mutex::Lock _l(_lock); @@ -755,57 +752,55 @@ int Network::filterIncomingPacket( const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { - case DOZTFILTER_NO_MATCH: - break; + + case DOZTFILTER_NO_MATCH: { + Membership::CapabilityIterator mci(m); + const Capability *c; + while ((c = mci.next(_config))) { + ztDest2 = ztDest; // sanity check + Address cc2; + unsigned int ccLength2 = 0; + switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + case DOZTFILTER_NO_MATCH: + case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern + break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest will have been changed in _doZtFilter() + case DOZTFILTER_ACCEPT: + accept = 1; // ACCEPT + break; + case DOZTFILTER_SUPER_ACCEPT: + accept = 2; // super-ACCEPT + break; + } + if (accept) { + if (cc2) { + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); + outp.append(_id); + outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + macDest.appendTo(outp); + macSource.appendTo(outp); + outp.append((uint16_t)etherType); + outp.append(frameData,ccLength2); + outp.compress(); + RR->sw->send(outp,true); + } + break; + } + } + } break; + case DOZTFILTER_DROP: return 0; // DROP + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: - mainRuleTableMatch = true; accept = 1; // ACCEPT break; case DOZTFILTER_SUPER_ACCEPT: - mainRuleTableMatch = true; accept = 2; // super-ACCEPT break; } - if (!mainRuleTableMatch) { - Membership::CapabilityIterator mci(m); - const Capability *c; - while ((c = mci.next(_config))) { - ztDest2 = ztDest; // sanity check - Address cc2; - unsigned int ccLength2 = 0; - switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { - case DOZTFILTER_NO_MATCH: - case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern - break; - case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest will have been changed in _doZtFilter() - case DOZTFILTER_ACCEPT: - accept = 1; // ACCEPT - break; - case DOZTFILTER_SUPER_ACCEPT: - accept = 2; // super-ACCEPT - break; - } - if (accept) { - if (cc2) { - Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); - outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 - macDest.appendTo(outp); - macSource.appendTo(outp); - outp.append((uint16_t)etherType); - outp.append(frameData,ccLength2); - outp.compress(); - RR->sw->send(outp,true); - } - break; - } - } - } - if (accept) { if (cc) { Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); -- cgit v1.2.3 From 25056de5d3845370366114782d2611ca9a139042 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 Aug 2016 17:56:59 -0700 Subject: Also need to send credentials when TEEing and REDIRECTing. --- node/Membership.hpp | 2 +- node/Network.cpp | 31 ++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/node/Membership.hpp b/node/Membership.hpp index 90363e2e..5e5efc50 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -124,7 +124,7 @@ public: * * @param RR Runtime environment * @param now Current time - * @param peerAddress Address of member peer + * @param peerAddress Address of member peer (the one that this Membership describes) * @param nconf My network config * @param cap Capability to send or 0 if none */ diff --git a/node/Network.cpp b/node/Network.cpp index 13566a80..9464c186 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -663,10 +663,16 @@ bool Network::filterOutgoingPacket( case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + relevantCap = &(_config.capabilities[c]); + accept = true; + if ((!noTee)&&(cc2)) { + _memberships[cc2].sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,relevantCap); + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 @@ -677,8 +683,7 @@ bool Network::filterOutgoingPacket( outp.compress(); RR->sw->send(outp,true); } - relevantCap = &(_config.capabilities[c]); - accept = true; + break; } if (accept) @@ -697,10 +702,9 @@ bool Network::filterOutgoingPacket( } if (accept) { - if (ztDest2) - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); - if ((!noTee)&&(cc)) { + _memberships[cc].sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,relevantCap); + Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 @@ -712,7 +716,9 @@ bool Network::filterOutgoingPacket( RR->sw->send(outp,true); } - if (ztDest != ztDest2) { + if ((ztDest != ztDest2)&&(ztDest2)) { + _memberships[ztDest2].sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); + Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 @@ -722,7 +728,10 @@ bool Network::filterOutgoingPacket( outp.append(frameData,frameLen); outp.compress(); RR->sw->send(outp,true); + return false; // DROP locally, since we redirected + } else if (ztDest) { + m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,relevantCap); } } @@ -772,8 +781,11 @@ int Network::filterIncomingPacket( accept = 2; // super-ACCEPT break; } + if (accept) { if (cc2) { + _memberships[cc2].sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,(const Capability *)0); + Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 @@ -803,6 +815,8 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { + _memberships[cc].sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,(const Capability *)0); + Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 @@ -814,7 +828,9 @@ int Network::filterIncomingPacket( RR->sw->send(outp,true); } - if (ztDest != ztDest2) { + if ((ztDest != ztDest2)&&(ztDest2)) { + _memberships[ztDest2].sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,(const Capability *)0); + Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 @@ -824,6 +840,7 @@ int Network::filterIncomingPacket( outp.append(frameData,frameLen); outp.compress(); RR->sw->send(outp,true); + return 0; // DROP locally, since we redirected } } -- cgit v1.2.3 From 8b6d23b9f6bdb9f2d831a0e43a189a10a57f7359 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Sep 2016 12:07:17 -0700 Subject: Optimize filter code a bit, and add a network-level setting for what should happen if an unsupported or unknown MATCH is encountered in a rules table. --- include/ZeroTierOne.h | 20 ++++--- node/Network.cpp | 140 +++++++++++++++++++++++++------------------------ node/NetworkConfig.hpp | 5 ++ 3 files changed, 89 insertions(+), 76 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 73450006..db0560a3 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -483,18 +483,14 @@ enum ZT_VirtualNetworkType ZT_NETWORK_TYPE_PUBLIC = 1 }; -/* - - TEE : should use a field to indicate how many bytes of each packet max are TEE'd - - Controller : web hooks for auth, optional required re-auth? or auth for a period of time? auto-expiring auth? -*/ - /** * The type of a virtual network rules table entry * * These must range from 0 to 127 (0x7f) because the most significant bit * is reserved as a NOT flag. * - * Each rule is composed of one or more MATCHes followed by an ACTION. + * Each rule is composed of zero or more MATCHes followed by an ACTION. + * An ACTION with no MATCHes is always taken. */ enum ZT_VirtualNetworkRuleType { @@ -525,6 +521,11 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 4, + /** + * Maximum ID for an ACTION, anything higher is a MATCH + */ + ZT_NETWORK_RULE_ACTION__MAX_ID = 31, + // 32 to 127 reserved for match criteria /** @@ -640,7 +641,12 @@ enum ZT_VirtualNetworkRuleType /** * Match if local and remote tags XORed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 54 + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 54, + + /** + * Maximum ID allowed for a MATCH entry in the rules table + */ + ZT_NETWORK_RULE_MATCH__MAX_ID = 127 }; /** diff --git a/node/Network.cpp b/node/Network.cpp index 9464c186..7aaf6933 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -35,7 +35,7 @@ #include "Node.hpp" #include "Peer.hpp" -// Uncomment to enable ZT_NETWORK_RULE_ACTION_DEBUG_LOG rule output to STDOUT +// Uncomment to make the rules engine dump trace info to stdout #define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { @@ -73,7 +73,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: return "MATCH_TAGS_BITWISE_AND"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: return "MATCH_TAGS_BITWISE_OR"; case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: return "MATCH_TAGS_BITWISE_XOR"; - default: return "BAD_RULE_TYPE"; + default: return "???"; } } static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) @@ -172,110 +172,110 @@ static _doZtFilterResult _doZtFilter( Address &cc, // MUTABLE unsigned int &ccLength) // MUTABLE { - // For each set of rules we start by assuming that they match (since no constraints - // yields a 'match all' rule). - uint8_t thisSetMatches = 1; - #ifdef ZT_RULES_ENGINE_DEBUGGING + char dpbuf[1024]; // used by FILTER_TRACE macro std::vector dlog; - char dpbuf[1024]; #endif // ZT_RULES_ENGINE_DEBUGGING + // The default match state for each set of entries starts as 'true' since an + // ACTION with no MATCH entries preceding it is always taken. + uint8_t thisSetMatches = 1; + for(unsigned int rn=0;rnidentity.address()) { + if (inbound) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); - dlog.clear(); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"interpreted as super-ACCEPT on inbound since we are target"); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; - } - continue; - - case ZT_NETWORK_RULE_ACTION_TEE: - case ZT_NETWORK_RULE_ACTION_REDIRECT: { - const Address fwdAddr(rules[rn].v.fwd.address); - if (fwdAddr == ztSource) { + return DOZTFILTER_SUPER_ACCEPT; + } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored since source is target"); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"skipped as no-op on outbound since we are target"); + dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; - } else if (fwdAddr == RR->identity.address()) { - if (inbound) { + } + } else if (fwdAddr == ztDest) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT interpreted as super-accept since we are target"); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"skipped as no-op because destination is already target"); + dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - return DOZTFILTER_SUPER_ACCEPT; - } else { + } else { + if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored on outbound since we are target"); + _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; - } - } else if (fwdAddr == ztDest) { + ztDest = fwdAddr; + return DOZTFILTER_REDIRECT; + } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"TEE/REDIRECT ignored since destination is target"); + _dumpFilterTrace("ACTION_TEE",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; - } else { - if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { + cc = fwdAddr; + ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; + } + } + } continue; + + // This is a no-op that exists for use with rules engine tracing and isn't for use in production + case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: // a no-op target specifically for debugging purposes #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - ztDest = fwdAddr; - return DOZTFILTER_REDIRECT; - } else { + continue; + + // Unrecognized ACTIONs are ignored as no-ops + default: #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_TEE",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - cc = fwdAddr; - ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; - thisSetMatches = 1; - } + continue; } - } continue; - - case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: // a no-op target specifically for debugging purposes + } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - thisSetMatches = 1; + thisSetMatches = 1; // reset to default true for next batch of entries continue; - - default: break; + } } - // No need to evaluate MATCH entries beyond where thisSetMatches is no longer still true + // Circuit breaker: skip further MATCH entries up to next ACTION if match state is false if (!thisSetMatches) continue; + // If this was not an ACTION evaluate next MATCH and update thisSetMatches with (AND [result]) uint8_t thisRuleMatches = 0; - switch(rt) { case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); @@ -553,12 +553,14 @@ static _doZtFilterResult _doZtFilter( } } break; - default: // rules we don't know do not match -- this means upgrading may be necessary before shipping new rules on a network or old clients might get blocked - thisRuleMatches = 0; + // The result of an unsupported MATCH is configurable at the network + // level via a flag. + default: + thisRuleMatches = (uint8_t)((nconf.flags & ZT_NETWORKCONFIG_FLAG_RULES_RESULT_OF_UNSUPPORTED_MATCH) != 0); break; } - // thisSetMatches remains true if the current rule matched (or did NOT match if NOT bit is set) + // State of equals state AND result of last MATCH (possibly NOTed depending on bit 0x80) thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 67126d64..e2bacb07 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -71,6 +71,11 @@ */ #define ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION 0x0000000000000004ULL +/** + * Flag: result of unrecognized MATCH entries in a rules table: match if set, no-match if clear + */ +#define ZT_NETWORKCONFIG_FLAG_RULES_RESULT_OF_UNSUPPORTED_MATCH 0x0000000000000008ULL + /** * Device is an active bridge */ -- cgit v1.2.3 From 22271f2a490a1f3c2ade184d758cadfe0e461544 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Sep 2016 13:36:41 -0700 Subject: Cleanup. --- node/Network.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 7aaf6933..4dc8aa30 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -40,6 +40,8 @@ namespace ZeroTier { +namespace { + #ifdef ZT_RULES_ENGINE_DEBUGGING #define FILTER_TRACE(f,...) { Utils::snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) @@ -144,11 +146,11 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig enum _doZtFilterResult { - DOZTFILTER_NO_MATCH = 0, - DOZTFILTER_DROP = 1, - DOZTFILTER_REDIRECT = 2, - DOZTFILTER_ACCEPT = 3, - DOZTFILTER_SUPER_ACCEPT = 4 + DOZTFILTER_NO_MATCH, + DOZTFILTER_DROP, + DOZTFILTER_REDIRECT, + DOZTFILTER_ACCEPT, + DOZTFILTER_SUPER_ACCEPT }; static _doZtFilterResult _doZtFilter( const RuntimeEnvironment *RR, @@ -567,6 +569,8 @@ static _doZtFilterResult _doZtFilter( return DOZTFILTER_NO_MATCH; } +} // anonymous namespace + const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : -- cgit v1.2.3 From d5e6f59004458ee2bddcaa18c6d0a8dfae2a2fc3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Sep 2016 13:45:32 -0700 Subject: . --- node/Switch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 8e41c89f..aae16001 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -94,7 +94,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } } - } else if (len > ZT_PROTO_MIN_FRAGMENT_LENGTH) { // min length check is important! + } else if (len > ZT_PROTO_MIN_FRAGMENT_LENGTH) { // SECURITY: min length check is important since we do some C-style stuff below! if (reinterpret_cast(data)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR) { // Handle fragment ---------------------------------------------------- -- cgit v1.2.3 From a3bdae9735572fd6d888e6bf29b0302983a7c24c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Sep 2016 15:43:07 -0700 Subject: Work in progress: Path canonicalization refactor. --- node/Constants.hpp | 5 -- node/IncomingPacket.hpp | 38 ++------- node/Path.cpp | 2 +- node/Path.hpp | 214 +++++++++++++++++++----------------------------- node/Switch.cpp | 11 ++- node/Topology.hpp | 21 +++++ 6 files changed, 120 insertions(+), 171 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index b9308abd..8a596fb3 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -269,11 +269,6 @@ */ #define ZT_PEER_ACTIVITY_TIMEOUT 500000 -/** - * Timeout for path activity - */ -#define ZT_PATH_ACTIVITY_TIMEOUT ZT_PEER_ACTIVITY_TIMEOUT - /** * No answer timeout to trigger dead path detection */ diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 303ac5f8..35438f4f 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -22,7 +22,7 @@ #include #include "Packet.hpp" -#include "InetAddress.hpp" +#include "Path.hpp" #include "Utils.hpp" #include "MulticastGroup.hpp" #include "Peer.hpp" @@ -56,41 +56,24 @@ class IncomingPacket : public Packet public: IncomingPacket() : Packet(), - _receiveTime(0), - _localAddress(), - _remoteAddress() + _receiveTime(0) { } - IncomingPacket(const IncomingPacket &p) - { - // All fields including InetAddress are memcpy'able - memcpy(this,&p,sizeof(IncomingPacket)); - } - /** * Create a new packet-in-decode * * @param data Packet data * @param len Packet length - * @param localAddress Local interface address - * @param remoteAddress Address from which packet came + * @param path Path over which packet arrived * @param now Current time * @throws std::out_of_range Range error processing packet */ - IncomingPacket(const void *data,unsigned int len,const InetAddress &localAddress,const InetAddress &remoteAddress,uint64_t now) : + IncomingPacket(const void *data,unsigned int len,const SharedPtr &path,uint64_t now) : Packet(data,len), _receiveTime(now), - _localAddress(localAddress), - _remoteAddress(remoteAddress) - { - } - - inline IncomingPacket &operator=(const IncomingPacket &p) + _path(path) { - // All fields including InetAddress are memcpy'able - memcpy(this,&p,sizeof(IncomingPacket)); - return *this; } /** @@ -98,17 +81,15 @@ public: * * @param data Packet data * @param len Packet length - * @param localAddress Local interface address - * @param remoteAddress Address from which packet came + * @param path Path over which packet arrived * @param now Current time * @throws std::out_of_range Range error processing packet */ - inline void init(const void *data,unsigned int len,const InetAddress &localAddress,const InetAddress &remoteAddress,uint64_t now) + inline void init(const void *data,unsigned int len,const SharedPtr &path,uint64_t now) { copyFrom(data,len); _receiveTime = now; - _localAddress = localAddress; - _remoteAddress = remoteAddress; + _path = path; } /** @@ -174,8 +155,7 @@ private: bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr &peer); uint64_t _receiveTime; - InetAddress _localAddress; - InetAddress _remoteAddress; + SharedPtr _path; }; } // namespace ZeroTier diff --git a/node/Path.cpp b/node/Path.cpp index 5692af66..5592bacc 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -25,7 +25,7 @@ namespace ZeroTier { bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now) { if (RR->node->putPacket(_localAddress,address(),data,len)) { - sent(now); + _lastOut = now; return true; } return false; diff --git a/node/Path.hpp b/node/Path.hpp index ca5dd98f..f8d84d4b 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -27,27 +27,8 @@ #include "Constants.hpp" #include "InetAddress.hpp" - -// Note: if you change these flags check the logic below. Some of it depends -// on these bits being what they are. - -/** - * Flag indicating that this path is suboptimal - * - * Clusters set this flag on remote paths if GeoIP or other routing decisions - * indicate that a peer should be handed off to another cluster member. - */ -#define ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL 0x0001 - -/** - * Flag indicating that this path is optimal - * - * Peers set this flag on paths that are pushed by a cluster and indicated as - * optimal. A second flag is needed since we want to prioritize cluster optimal - * paths and de-prioritize sub-optimal paths and for new paths we don't know - * which one they are. So we want a trinary state: optimal, suboptimal, unknown. - */ -#define ZT_PATH_FLAG_CLUSTER_OPTIMAL 0x0002 +#include "SharedPtr.hpp" +#include "AtomicCounter.hpp" /** * Maximum return value of preferenceRank() @@ -59,34 +40,83 @@ namespace ZeroTier { class RuntimeEnvironment; /** - * Base class for paths - * - * The base Path class is an immutable value. + * A path across the physical network */ class Path { + friend class SharedPtr; + public: + /** + * Efficient unique key for paths in a Hashtable + */ + class HashKey + { + public: + HashKey() {} + + HashKey(const InetAddress &l,const InetAddress &r) + { + // This is an ad-hoc bit packing algorithm to yield unique keys for + // remote addresses and their local-side counterparts if defined. + // Portability across runtimes is not needed. + if (r.ss_family == AF_INET) { + _k[0] = (uint64_t)reinterpret_cast(&r)->sin_addr.s_addr; + _k[1] = (uint64_t)reinterpret_cast(&r)->sin_port; + if (l.ss_family == AF_INET) { + _k[2] = (uint64_t)reinterpret_cast(&l)->sin_addr.s_addr; + _k[3] = (uint64_t)reinterpret_cast(&r)->sin_port; + } else { + _k[2] = 0; + _k[3] = 0; + } + } else if (r.ss_family == AF_INET6) { + const uint8_t *a = reinterpret_cast(reinterpret_cast(&r)->sin6_addr.s6_addr); + uint8_t *b = reinterpret_cast(_k); + for(unsigned int i=0;i<16;++i) b[i] = a[i]; + _k[2] = ~((uint64_t)reinterpret_cast(&r)->sin6_port); + if (l.ss_family == AF_INET6) { + _k[2] ^= ((uint64_t)reinterpret_cast(&r)->sin6_port) << 32; + a = reinterpret_cast(reinterpret_cast(&l)->sin6_addr.s6_addr); + b += 24; + for(unsigned int i=0;i<8;++i) b[i] = a[i]; + a += 8; + for(unsigned int i=0;i<8;++i) b[i] ^= a[i]; + } + } else { + _k[0] = 0; + _k[1] = 0; + _k[2] = 0; + _k[3] = 0; + } + } + + inline unsigned long hashCode() const { return (unsigned long)(_k[0] + _k[1] + _k[2] + _k[3]); } + + inline bool operator==(const HashKey &k) const { return ( (_k[0] == k._k[0]) && (_k[1] == k._k[1]) && (_k[2] == k._k[2]) && (_k[3] == k._k[3]) ); } + inline bool operator!=(const HashKey &k) const { return (!(*this == k)); } + + private: + uint64_t _k[4]; + }; + Path() : - _lastSend(0), - _lastPing(0), - _lastKeepalive(0), - _lastReceived(0), + _lastOut(0), + _lastIn(0), _addr(), _localAddress(), - _flags(0), - _ipScope(InetAddress::IP_SCOPE_NONE) + _ipScope(InetAddress::IP_SCOPE_NONE), + _clusterSuboptimal(false) { } Path(const InetAddress &localAddress,const InetAddress &addr) : - _lastSend(0), - _lastPing(0), - _lastKeepalive(0), - _lastReceived(0), + _lastOut(0), + _lastIn(0), _addr(addr), _localAddress(localAddress), - _flags(0), - _ipScope(addr.ipScope()) + _ipScope(addr.ipScope()), + _clusterSuboptimal(false) { } @@ -104,44 +134,17 @@ public: * * @param t Time of send */ - inline void sent(uint64_t t) { _lastSend = t; } - - /** - * Called when we've sent a ping or echo - * - * @param t Time of send - */ - inline void pinged(uint64_t t) { _lastPing = t; } + inline void sent(const uint64_t t) { _lastOut = t; } /** - * Called when we send a NAT keepalive - * - * @param t Time of send - */ - inline void sentKeepalive(uint64_t t) { _lastKeepalive = t; } - - /** - * Called when a packet is received from this remote path + * Called when a packet is received from this remote path, regardless of content * * @param t Time of receive */ - inline void received(uint64_t t) - { - _lastReceived = t; - _probation = 0; - } + inline void received(const uint64_t t) { _lastIn = t; } /** - * @param now Current time - * @return True if this path appears active - */ - inline bool active(uint64_t now) const - { - return ( ((now - _lastReceived) < ZT_PATH_ACTIVITY_TIMEOUT) && (_probation < ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION) ); - } - - /** - * Send a packet via this path + * Send a packet via this path (last out time is also updated) * * @param RR Runtime environment * @param data Packet data @@ -156,26 +159,6 @@ public: */ inline const InetAddress &localAddress() const throw() { return _localAddress; } - /** - * @return Time of last send to this path - */ - inline uint64_t lastSend() const throw() { return _lastSend; } - - /** - * @return Time we last pinged or dead path checked this link - */ - inline uint64_t lastPing() const throw() { return _lastPing; } - - /** - * @return Time of last keepalive - */ - inline uint64_t lastKeepalive() const throw() { return _lastKeepalive; } - - /** - * @return Time of last receive from this path - */ - inline uint64_t lastReceived() const throw() { return _lastReceived; } - /** * @return Physical address */ @@ -187,26 +170,19 @@ public: inline InetAddress::IpScope ipScope() const throw() { return _ipScope; } /** - * @param f Valuve of ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL and inverse of ZT_PATH_FLAG_CLUSTER_OPTIMAL (both are changed) + * @param f Is this path cluster-suboptimal? */ - inline void setClusterSuboptimal(bool f) - { - if (f) { - _flags = (_flags | ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL) & ~ZT_PATH_FLAG_CLUSTER_OPTIMAL; - } else { - _flags = (_flags | ZT_PATH_FLAG_CLUSTER_OPTIMAL) & ~ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL; - } - } + inline void setClusterSuboptimal(const bool f) { _clusterSuboptimal = f; } /** - * @return True if ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL is set + * @return True if cluster-suboptimal (for someone) */ - inline bool isClusterSuboptimal() const { return ((_flags & ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL) != 0); } + inline bool isClusterSuboptimal() const { return _clusterSuboptimal; } /** - * @return True if ZT_PATH_FLAG_CLUSTER_OPTIMAL is set + * @return True if cluster-optimal (for someone) (the default) */ - inline bool isClusterOptimal() const { return ((_flags & ZT_PATH_FLAG_CLUSTER_OPTIMAL) != 0); } + inline bool isClusterOptimal() const { return (!(_clusterSuboptimal)); } /** * @return Preference rank, higher == better (will be less than 255) @@ -230,28 +206,17 @@ public: // This is a little bit convoluted because we try to be branch-free, using multiplication instead of branches for boolean flags // Start with the last time this path was active, and add a fudge factor to prevent integer underflow if _lastReceived is 0 - uint64_t score = _lastReceived + (ZT_PEER_DIRECT_PING_DELAY * (ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION + 1)); + uint64_t score = _lastIn + (ZT_PEER_DIRECT_PING_DELAY * (ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION + 1)); // Increase score based on path preference rank, which is based on IP scope and address family score += preferenceRank() * (ZT_PEER_DIRECT_PING_DELAY / ZT_PATH_MAX_PREFERENCE_RANK); - // Increase score if this is known to be an optimal path to a cluster - score += (uint64_t)(_flags & ZT_PATH_FLAG_CLUSTER_OPTIMAL) * (ZT_PEER_DIRECT_PING_DELAY / 2); // /2 because CLUSTER_OPTIMAL is flag 0x0002 - // Decrease score if this is known to be a sub-optimal path to a cluster - score -= (uint64_t)(_flags & ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL) * ZT_PEER_DIRECT_PING_DELAY; - - // Penalize for missed ECHO tests in dead path detection - score -= (uint64_t)((ZT_PEER_DIRECT_PING_DELAY / 2) * _probation); + score -= ((uint64_t)_clusterSuboptimal) * ZT_PEER_DIRECT_PING_DELAY; return score; } - /** - * @return True if address is non-NULL - */ - inline operator bool() const throw() { return (_addr); } - /** * Check whether this address is valid for a ZeroTier path * @@ -293,29 +258,14 @@ public: return false; } - /** - * @return Current path probation count (for dead path detect) - */ - inline unsigned int probation() const { return _probation; } - - /** - * Increase this path's probation violation count (for dead path detect) - */ - inline void increaseProbation() { ++_probation; } - - inline bool operator==(const Path &p) const { return ((p._addr == _addr)&&(p._localAddress == _localAddress)); } - inline bool operator!=(const Path &p) const { return ((p._addr != _addr)||(p._localAddress != _localAddress)); } - private: - uint64_t _lastSend; - uint64_t _lastPing; - uint64_t _lastKeepalive; - uint64_t _lastReceived; + uint64_t _lastOut; + uint64_t _lastIn; InetAddress _addr; InetAddress _localAddress; - unsigned int _flags; - unsigned int _probation; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often + AtomicCounter __refCount; + bool _clusterSuboptimal; }; } // namespace ZeroTier diff --git a/node/Switch.cpp b/node/Switch.cpp index aae16001..dc238607 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -73,6 +73,9 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from try { const uint64_t now = RR->node->now(); + SharedPtr path(RR->topology->getPath(localAddr,fromAddr)); + path->received(now); + if (len == 13) { /* LEGACY: before VERB_PUSH_DIRECT_PATHS, peers used broadcast * announcements on the LAN to solve the 'same network problem.' We @@ -90,7 +93,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from _lastBeaconResponse = now; Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP); outp.armor(peer->key(),true); - RR->node->putPacket(localAddr,fromAddr,outp.data(),outp.size()); + path->send(RR,outp.data(),outp.size(),now); } } @@ -259,7 +262,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // We have all fragments -- assemble and process full Packet //TRACE("packet %.16llx is complete, assembling and processing...",pid); - rq->frag0.init(data,len,localAddr,fromAddr,now); + rq->frag0.init(data,len,path,now); for(unsigned int f=1;ftotalFragments;++f) rq->frag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); @@ -270,12 +273,12 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } } else { // Still waiting on more fragments, but keep the head - rq->frag0.init(data,len,localAddr,fromAddr,now); + rq->frag0.init(data,len,path,now); } } // else this is a duplicate head, ignore } else { // Packet is unfragmented, so just process it - IncomingPacket packet(data,len,localAddr,fromAddr,now); + IncomingPacket packet(data,len,path,now); if (!packet.tryDecode(RR)) { Mutex::Lock _l(_rxQueue_m); RXQueueEntry *rq = &(_rxQueue[ZT_RX_QUEUE_SIZE - 1]); diff --git a/node/Topology.hpp b/node/Topology.hpp index b8213cf8..e63766cb 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -33,6 +33,7 @@ #include "Address.hpp" #include "Identity.hpp" #include "Peer.hpp" +#include "Path.hpp" #include "Mutex.hpp" #include "InetAddress.hpp" #include "Hashtable.hpp" @@ -89,6 +90,22 @@ public: return SharedPtr(); } + /** + * Get a Path object for a given local and remote physical address, creating if needed + * + * @param l Local address or NULL for 'any' or 'wildcard' + * @param r Remote address + * @return Pointer to canonicalized Path object + */ + inline SharedPtr getPath(const InetAddress &l,const InetAddress &r) + { + Mutex::Lock _l(_lock); + SharedPtr &p = _paths[Path::HashKey(l,r)]; + if (!p) + p.setToUnsafe(new Path(l,r)); + return p; + } + /** * Get the identity of a peer * @@ -319,8 +336,12 @@ private: uint64_t _trustedPathIds[ZT_MAX_TRUSTED_PATHS]; InetAddress _trustedPathNetworks[ZT_MAX_TRUSTED_PATHS]; unsigned int _trustedPathCount; + World _world; + Hashtable< Address,SharedPtr > _peers; + Hashtable< Path::HashKey,SharedPtr > _paths; + std::vector< Address > _rootAddresses; std::vector< SharedPtr > _rootPeers; bool _amRoot; -- cgit v1.2.3 From e8f6b4b5d33e7b762b952d599e8cc9e730b21c03 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 11:51:33 -0700 Subject: Rest of big Path canonicalization refactor. --- node/Constants.hpp | 36 ++--- node/IncomingPacket.cpp | 290 ++++++++++++++++++++-------------------- node/Node.cpp | 18 +-- node/Path.hpp | 82 ++++-------- node/Peer.cpp | 348 ++++++++++++++++++++++++++---------------------- node/Peer.hpp | 166 ++++++++++------------- node/Switch.cpp | 24 ++-- 7 files changed, 461 insertions(+), 503 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 8a596fb3..6d6f44e0 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -238,46 +238,38 @@ */ #define ZT_MULTICAST_TRANSMIT_TIMEOUT 5000 -/** - * Default maximum number of peers to address with a single multicast (if unspecified in network config) - */ -#define ZT_MULTICAST_DEFAULT_LIMIT 32 - -/** - * How frequently to send a zero-byte UDP keepalive packet - * - * There are NATs with timeouts as short as 20 seconds, so this turns out - * to be needed. - */ -#define ZT_NAT_KEEPALIVE_DELAY 19000 - /** * Delay between scans of the topology active peer DB for peers that need ping * * This is also how often pings will be retried to upstream peers (relays, roots) * constantly until something is heard. */ -#define ZT_PING_CHECK_INVERVAL 9500 +#define ZT_PING_CHECK_INVERVAL 8000 /** - * Delay between ordinary case pings of direct links + * How frequently to send heartbeats over in-use paths */ -#define ZT_PEER_DIRECT_PING_DELAY 60000 +#define ZT_PATH_HEARTBEAT_PERIOD 18000 /** - * Timeout for overall peer activity (measured from last receive) + * Paths are considered inactive if they have not received traffic in this long */ -#define ZT_PEER_ACTIVITY_TIMEOUT 500000 +#define ZT_PATH_ALIVE_TIMEOUT ((ZT_PATH_HEARTBEAT_PERIOD * 2) + 2000) /** - * No answer timeout to trigger dead path detection + * Delay between full-fledge pings of directly connected peers */ -#define ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT 2000 +#define ZT_PEER_PING_PERIOD 60000 /** - * Probation threshold after which a path becomes dead + * Peers forget paths that have not spoken in this long */ -#define ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION 3 +#define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 3) + 3000) + +/** + * Timeout for overall peer activity (measured from last receive) + */ +#define ZT_PEER_ACTIVITY_TIMEOUT 500000 /** * Delay between requests for updated network autoconf information diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 4b013078..fafd5679 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -54,11 +54,11 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) // If this is marked as a packet via a trusted path, check source address and path ID. // Obviously if no trusted paths are configured this always returns false and such // packets are dropped on the floor. - if (RR->topology->shouldInboundPathBeTrusted(_remoteAddress,trustedPathId())) { + if (RR->topology->shouldInboundPathBeTrusted(_path->address(),trustedPathId())) { trusted = true; - TRACE("TRUSTED PATH packet approved from %s(%s), trusted path ID %llx",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str(),trustedPathId()); + TRACE("TRUSTED PATH packet approved from %s(%s), trusted path ID %llx",sourceAddress.toString().c_str(),_path->address().toString().c_str(),trustedPathId()); } else { - TRACE("dropped packet from %s(%s), cipher set to trusted path mode but path %llx@%s is not trusted!",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str(),trustedPathId(),_remoteAddress.toString().c_str()); + TRACE("dropped packet from %s(%s), cipher set to trusted path mode but path %llx@%s is not trusted!",sourceAddress.toString().c_str(),_path->address().toString().c_str(),trustedPathId(),_path->address().toString().c_str()); return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { @@ -73,42 +73,42 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) if (peer) { if (!trusted) { if (!dearmor(peer->key())) { - TRACE("dropped packet from %s(%s), MAC authentication failed (size: %u)",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str(),size()); + TRACE("dropped packet from %s(%s), MAC authentication failed (size: %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); return true; } } if (!uncompress()) { - TRACE("dropped packet from %s(%s), compressed data invalid",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped packet from %s(%s), compressed data invalid",sourceAddress.toString().c_str(),_path->address().toString().c_str()); return true; } const Packet::Verb v = verb(); - //TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); + //TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_path->address().toString().c_str()); switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(_localAddress,_remoteAddress,hops(),packetId(),v,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; - case Packet::VERB_HELLO: return _doHELLO(RR,peer); - case Packet::VERB_ERROR: return _doERROR(RR,peer); - case Packet::VERB_OK: return _doOK(RR,peer); - case Packet::VERB_WHOIS: return _doWHOIS(RR,peer); - case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,peer); - case Packet::VERB_FRAME: return _doFRAME(RR,peer); - case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,peer); - case Packet::VERB_ECHO: return _doECHO(RR,peer); - case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); - case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); - case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); - case Packet::VERB_NETWORK_CONFIG_REFRESH: return _doNETWORK_CONFIG_REFRESH(RR,peer); - case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); - case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); - case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); - case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); - case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); - case Packet::VERB_REQUEST_PROOF_OF_WORK: return _doREQUEST_PROOF_OF_WORK(RR,peer); + case Packet::VERB_HELLO: return _doHELLO(RR,peer); + case Packet::VERB_ERROR: return _doERROR(RR,peer); + case Packet::VERB_OK: return _doOK(RR,peer); + case Packet::VERB_WHOIS: return _doWHOIS(RR,peer); + case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,peer); + case Packet::VERB_FRAME: return _doFRAME(RR,peer); + case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,peer); + case Packet::VERB_ECHO: return _doECHO(RR,peer); + case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); + case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); + case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); + case Packet::VERB_NETWORK_CONFIG_REFRESH: return _doNETWORK_CONFIG_REFRESH(RR,peer); + case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); + case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); + case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); + case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); + case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); + case Packet::VERB_REQUEST_PROOF_OF_WORK: return _doREQUEST_PROOF_OF_WORK(RR,peer); case Packet::VERB_USER_MESSAGE: return true; } @@ -119,7 +119,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } catch ( ... ) { // Exceptions are more informatively caught in _do...() handlers but // this outer try/catch will catch anything else odd. - TRACE("dropped ??? from %s(%s): unexpected exception in tryDecode()",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped ??? from %s(%s): unexpected exception in tryDecode()",sourceAddress.toString().c_str(),_path->address().toString().c_str()); return true; } } @@ -131,7 +131,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr const uint64_t inRePacketId = at(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID); const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE]; - //TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb)); + //TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); switch(errorCode) { @@ -172,9 +172,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr default: break; } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); + peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); } catch ( ... ) { - TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -216,11 +216,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } if (protoVersion < ZT_PROTO_VERSION_MIN) { - TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); return true; } if (fromAddress != id.address()) { - TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_path->address().toString().c_str()); return true; } @@ -235,18 +235,18 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { if (dearmor(key)) { // ensure packet is authentic, otherwise drop - TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_path->address().toString().c_str()); Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_HELLO); outp.append((uint64_t)pid); outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION); outp.armor(key,true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } else { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); } } else { - TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_path->address().toString().c_str()); } return true; @@ -254,7 +254,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer // Identity is the same as the one we already have -- check packet integrity if (!dearmor(peer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); return true; } @@ -265,14 +265,14 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer // Check identity proof of work if (!id.locallyValidate()) { - TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); return true; } // Check packet integrity and authentication SharedPtr newPeer(new Peer(RR,RR->identity,id)); if (!dearmor(newPeer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); return true; } peer = RR->topology->addPeer(newPeer); @@ -284,7 +284,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } if (externalSurfaceAddress) - RR->sa->iam(id.address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now()); + RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now()); Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_HELLO); @@ -295,7 +295,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR); outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); if (protoVersion >= 5) { - _remoteAddress.serialize(outp); + _path->address().serialize(outp); } else { /* LEGACY COMPATIBILITY HACK: * @@ -320,7 +320,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer * nulling out the port field. Since this info is only used for empirical * detection of link changes, it doesn't break anything else. */ - InetAddress tmpa(_remoteAddress); + InetAddress tmpa(_path->address()); tmpa.setPort(0); tmpa.serialize(outp); } @@ -336,12 +336,12 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -352,7 +352,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); - //TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb)); + //TRACE("%s(%s): OK(%s)",source().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); switch(inReVerb) { @@ -364,7 +364,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION); if (vProto < ZT_PROTO_VERSION_MIN) { - TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_path->address().toString().c_str()); return true; } @@ -386,13 +386,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p } } - TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); + TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); peer->addDirectLatencyMeasurment(latency); peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if (externalSurfaceAddress) - RR->sa->iam(peer->address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); + RR->sa->iam(peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; case Packet::VERB_WHOIS: { @@ -414,7 +414,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p totalSize = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen); chunkIndex = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen + 4); } - TRACE("%s(%s): OK(NETWORK_CONFIG_REQUEST) chunkLen==%u chunkIndex==%u totalSize==%u",source().toString().c_str(),_remoteAddress.toString().c_str(),chunkLen,chunkIndex,totalSize); + TRACE("%s(%s): OK(NETWORK_CONFIG_REQUEST) chunkLen==%u chunkIndex==%u totalSize==%u",source().toString().c_str(),_path->address().toString().c_str(),chunkLen,chunkIndex,totalSize); network->handleInboundConfigChunk(inRePacketId,chunkData,chunkLen,chunkIndex,totalSize); } } break; @@ -425,7 +425,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); - //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,mg.toString().c_str(),size()); + //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); } break; @@ -435,7 +435,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID); const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI)); - //TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),nwid,mg.toString().c_str(),flags); + //TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags); unsigned int offset = 0; @@ -461,9 +461,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); + peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { - TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -502,12 +502,12 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr if (count > 0) { outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -524,24 +524,22 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); if (!RR->topology->isUpstream(peer->identity())) { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); - } else if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) { - const uint64_t now = RR->node->now(); - peer->sendHELLO(_localAddress,atAddr,now,2); // send low-TTL packet to 'open' local NAT(s) - peer->sendHELLO(_localAddress,atAddr,now); + } else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { + RR->node->putPacket(_path->localAddress(),atAddr,"NATSUX",6,2); // send low-TTL packet to 'open' local NAT(s) + peer->sendHELLO(_path->localAddress(),atAddr,RR->node->now()); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } } else { - TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str()); } } else { - TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),with.toString().c_str()); + TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } - - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -553,8 +551,8 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr if (network) { if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { if (!network->isAllowed(peer)) { - TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,false); + TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,false); } else { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); const MAC sourceMac(peer->address(),network->id()); @@ -562,14 +560,14 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true); + peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true); } } } else { - TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); + TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } } catch ( ... ) { - TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -591,8 +589,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

isAllowed(peer)) { - TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -603,8 +601,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -614,14 +612,14 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (!network->config().permitsBridging(RR->identity.address())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } @@ -631,13 +629,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } } else { - TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); + TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } } catch ( ... ) { - TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -652,10 +650,10 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr if (size() > ZT_PACKET_IDX_PAYLOAD) outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); - peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); + peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -672,9 +670,9 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared RR->mc->add(now,nwid,group,peer->address()); } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -721,9 +719,9 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -744,7 +742,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons if (RR->localNetworkController) { NetworkConfig *netconf = new NetworkConfig(); try { - switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,*netconf)) { + switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) { case NetworkController::NETCONF_QUERY_OK: { netconfOk = true; @@ -783,7 +781,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); outp.append(nwid); outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } break; case NetworkController::NETCONF_QUERY_ACCESS_DENIED: { @@ -793,7 +791,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); outp.append(nwid); outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } break; case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR: @@ -816,16 +814,16 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); outp.append(nwid); outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); - TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what()); + TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { fprintf(stderr,"WARNING: network config request failed with exception: unknown exception" ZT_EOL_S); - TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -840,8 +838,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons if (network) { network->requestConfiguration(); } else { - TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); + TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); return true; } @@ -853,9 +851,9 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons } } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -868,7 +866,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); const unsigned int gatherLimit = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); - //TRACE("<address().toString().c_str(),gatherLimit,nwid,mg.toString().c_str()); if ((flags & 0x01) != 0) { try { @@ -880,7 +878,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar network->addCredential(com); } } catch ( ... ) { - TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str()); } } @@ -892,9 +890,9 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar mg.mac().appendTo(outp); outp.append((uint32_t)mg.adi()); const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); - if (gatheredLocally) { + if (gatheredLocally > 0) { outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } // If we are a member of a cluster, distribute this GATHER across it @@ -904,9 +902,9 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -933,8 +931,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // Check membership after we've read any included COM, since // that cert might be what we needed. if (!network->isAllowed(peer)) { - TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -960,13 +958,13 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { - TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { - TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -974,8 +972,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } @@ -996,16 +994,16 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share outp.append((unsigned char)0x02); // flag 0x02 = contains gather results if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) { outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); } else { - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { - TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -1017,8 +1015,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha // First, subject this to a rate limit if (!peer->shouldRespondToDirectPathPush(now)) { - TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1044,12 +1042,12 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha bool redundant = false; if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->setClusterOptimalPathForAddressFamily(a); + peer->makeExclusive(a); } else { redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->sendHELLO(InetAddress(),a,now); @@ -1063,12 +1061,12 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha bool redundant = false; if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->setClusterOptimalPathForAddressFamily(a); + peer->makeExclusive(a); } else { redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->sendHELLO(InetAddress(),a,now); @@ -1081,9 +1079,9 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha ptr += addrLen; } - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -1124,8 +1122,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt // Verify signature -- only tests signed by their originators are allowed const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { - TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } vlf += signatureLength; @@ -1141,13 +1139,13 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt if (originatorCredentialNetworkId) { SharedPtr network(RR->node->network(originatorCredentialNetworkId)); if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } } else { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } @@ -1165,9 +1163,9 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt remainingHopsPtr += ZT_ADDRESS_LENGTH; SharedPtr nhp(RR->topology->getPeer(nextHop[h])); if (nhp) { - Path *const rp = nhp->getBestPath(now); - if (rp) - nextHopBestPathAddress[h] = rp->address(); + SharedPtr nhbp(nhp->getBestPath(now,false)); + if (nhbp) + nextHopBestPathAddress[h] = nhbp->address(); } } } @@ -1190,8 +1188,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt outp.append((uint64_t)packetId()); peer->address().appendTo(outp); outp.append((uint8_t)hops()); - _localAddress.serialize(outp); - _remoteAddress.serialize(outp); + _path->localAddress().serialize(outp); + _path->address().serialize(outp); outp.append((uint16_t)0); // no additional fields outp.append((uint8_t)breadth); for(unsigned int h=0;hreceived(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -1263,9 +1261,9 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S } RR->node->postCircuitTestReport(&report); - peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -1308,28 +1306,28 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const outp.append((uint16_t)sizeof(result)); outp.append(result,sizeof(result)); outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } else { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_REQUEST_PROOF_OF_WORK); outp.append(pid); outp.append((unsigned char)Packet::ERROR_INVALID_REQUEST); outp.armor(peer->key(),true); - RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size()); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } } break; default: - TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unrecognized proof of work type",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unrecognized proof of work type",peer->address().toString().c_str(),_path->address().toString().c_str()); break; } - peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false); } else { - TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_path->address().toString().c_str()); } } catch ( ... ) { - TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str()); + TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } diff --git a/node/Node.cpp b/node/Node.cpp index ff564eee..39e24325 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -415,16 +415,16 @@ ZT_PeerList *Node::peers() const p->latency = pi->second->latency(); p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF; - std::vector paths(pi->second->paths()); - Path *bestPath = pi->second->getBestPath(_now); + std::vector< SharedPtr > paths(pi->second->paths()); + SharedPtr bestp(pi->second->getBestPath(_now,true)); p->pathCount = 0; - for(std::vector::iterator path(paths.begin());path!=paths.end();++path) { - memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage)); - p->paths[p->pathCount].lastSend = path->lastSend(); - p->paths[p->pathCount].lastReceive = path->lastReceived(); - p->paths[p->pathCount].active = path->active(_now) ? 1 : 0; - p->paths[p->pathCount].preferred = ((bestPath)&&(*path == *bestPath)) ? 1 : 0; - p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->address()); + for(std::vector< SharedPtr >::iterator path(paths.begin());path!=paths.end();++path) { + memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); + p->paths[p->pathCount].lastSend = (*path)->lastOut(); + p->paths[p->pathCount].lastReceive = (*path)->lastIn(); + p->paths[p->pathCount].active = (*path)->alive(_now) ? 1 : 0; + p->paths[p->pathCount].preferred = (*path == bestp) ? 1 : 0; + p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); ++p->pathCount; } } diff --git a/node/Path.hpp b/node/Path.hpp index f8d84d4b..68a630c3 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -105,8 +105,7 @@ public: _lastIn(0), _addr(), _localAddress(), - _ipScope(InetAddress::IP_SCOPE_NONE), - _clusterSuboptimal(false) + _ipScope(InetAddress::IP_SCOPE_NONE) { } @@ -115,27 +114,10 @@ public: _lastIn(0), _addr(addr), _localAddress(localAddress), - _ipScope(addr.ipScope()), - _clusterSuboptimal(false) + _ipScope(addr.ipScope()) { } - inline Path &operator=(const Path &p) - { - if (this != &p) - memcpy(this,&p,sizeof(Path)); - return *this; - } - - /** - * Called when a packet is sent to this remote path - * - * This is called automatically by Path::send(). - * - * @param t Time of send - */ - inline void sent(const uint64_t t) { _lastOut = t; } - /** * Called when a packet is received from this remote path, regardless of content * @@ -157,37 +139,22 @@ public: /** * @return Address of local side of this path or NULL if unspecified */ - inline const InetAddress &localAddress() const throw() { return _localAddress; } + inline const InetAddress &localAddress() const { return _localAddress; } /** * @return Physical address */ - inline const InetAddress &address() const throw() { return _addr; } + inline const InetAddress &address() const { return _addr; } /** * @return IP scope -- faster shortcut for address().ipScope() */ - inline InetAddress::IpScope ipScope() const throw() { return _ipScope; } - - /** - * @param f Is this path cluster-suboptimal? - */ - inline void setClusterSuboptimal(const bool f) { _clusterSuboptimal = f; } - - /** - * @return True if cluster-suboptimal (for someone) - */ - inline bool isClusterSuboptimal() const { return _clusterSuboptimal; } - - /** - * @return True if cluster-optimal (for someone) (the default) - */ - inline bool isClusterOptimal() const { return (!(_clusterSuboptimal)); } + inline InetAddress::IpScope ipScope() const { return _ipScope; } /** * @return Preference rank, higher == better (will be less than 255) */ - inline unsigned int preferenceRank() const throw() + inline unsigned int preferenceRank() const { /* First, since the scope enum values in InetAddress.hpp are in order of * use preference rank, we take that. Then we multiple by two, yielding @@ -201,20 +168,9 @@ public: /** * @return This path's overall quality score (higher is better) */ - inline uint64_t score() const throw() + inline uint64_t score() const { - // This is a little bit convoluted because we try to be branch-free, using multiplication instead of branches for boolean flags - - // Start with the last time this path was active, and add a fudge factor to prevent integer underflow if _lastReceived is 0 - uint64_t score = _lastIn + (ZT_PEER_DIRECT_PING_DELAY * (ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION + 1)); - - // Increase score based on path preference rank, which is based on IP scope and address family - score += preferenceRank() * (ZT_PEER_DIRECT_PING_DELAY / ZT_PATH_MAX_PREFERENCE_RANK); - - // Decrease score if this is known to be a sub-optimal path to a cluster - score -= ((uint64_t)_clusterSuboptimal) * ZT_PEER_DIRECT_PING_DELAY; - - return score; + return (_lastIn + (preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK))); } /** @@ -227,7 +183,6 @@ public: * @return True if address is good for ZeroTier path use */ static inline bool isAddressValidForPath(const InetAddress &a) - throw() { if ((a.ss_family == AF_INET)||(a.ss_family == AF_INET6)) { switch(a.ipScope()) { @@ -258,6 +213,26 @@ public: return false; } + /** + * @return True if path appears alive + */ + inline bool alive(const uint64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); } + + /** + * @return True if this path needs a heartbeat + */ + inline bool needsHeartbeat(const uint64_t now) const { return ((now - _lastOut) > ZT_PATH_HEARTBEAT_PERIOD); } + + /** + * @return Last time we sent something + */ + inline uint64_t lastOut() const { return _lastOut; } + + /** + * @return Last time we received anything + */ + inline uint64_t lastIn() const { return _lastIn; } + private: uint64_t _lastOut; uint64_t _lastIn; @@ -265,7 +240,6 @@ private: InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often AtomicCounter __refCount; - bool _clusterSuboptimal; }; } // namespace ZeroTier diff --git a/node/Peer.cpp b/node/Peer.cpp index 01492be1..251c5a5f 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -27,8 +27,6 @@ #include "Cluster.hpp" #include "Packet.hpp" -#include - #define ZT_PEER_PATH_SORT_INTERVAL 5000 namespace ZeroTier { @@ -45,7 +43,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastAnnouncedTo(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), - _lastPathSort(0), _vProto(0), _vMajor(0), _vMinor(0), @@ -60,8 +57,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident } void Peer::received( - const InetAddress &localAddr, - const InetAddress &remoteAddr, + const SharedPtr &path, unsigned int hops, uint64_t packetId, Packet::Verb verb, @@ -69,13 +65,15 @@ void Peer::received( Packet::Verb inReVerb, const bool trustEstablished) { + const uint64_t now = RR->node->now(); + #ifdef ZT_ENABLE_CLUSTER bool suboptimalPath = false; if ((RR->cluster)&&(hops == 0)) { // Note: findBetterEndpoint() is first since we still want to check // for a better endpoint even if we don't actually send a redirect. InetAddress redirectTo; - if ( (verb != Packet::VERB_OK) && (verb != Packet::VERB_ERROR) && (verb != Packet::VERB_RENDEZVOUS) && (verb != Packet::VERB_PUSH_DIRECT_PATHS) && (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),remoteAddr,false)) ) { + if ( (verb != Packet::VERB_OK) && (verb != Packet::VERB_ERROR) && (verb != Packet::VERB_RENDEZVOUS) && (verb != Packet::VERB_PUSH_DIRECT_PATHS) && (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),path->address(),false)) ) { if (_vProto >= 5) { // For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); @@ -93,7 +91,7 @@ void Peer::received( } outp.append((uint16_t)redirectTo.port()); outp.armor(_key,true); - RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size()); + path->send(RR,outp.data(),outp.size(),now); } else { // For older peers we use RENDEZVOUS to coax them into contacting us elsewhere. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS); @@ -108,14 +106,13 @@ void Peer::received( outp.append(redirectTo.rawIpData(),16); } outp.armor(_key,true); - RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size()); + path->send(RR,outp.data(),outp.size(),now); } suboptimalPath = true; } } #endif - const uint64_t now = RR->node->now(); _lastReceive = now; if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME)) _lastUnicastFrame = now; @@ -124,53 +121,47 @@ void Peer::received( if (hops == 0) { bool pathIsConfirmed = false; - unsigned int np = _numPaths; - for(unsigned int p=0;pnode->shouldUsePathForZeroTierTraffic(localAddr,remoteAddr))) { + if ((!pathIsConfirmed)&&(RR->node->shouldUsePathForZeroTierTraffic(path->localAddress(),path->address()))) { if (verb == Packet::VERB_OK) { + Mutex::Lock _l(_paths_m); - Path *slot = (Path *)0; - if (np < ZT_MAX_PEER_NETWORK_PATHS) { - slot = &(_paths[np++]); + unsigned int slot = 0; + if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) { + slot = _numPaths++; } else { - uint64_t slotWorstScore = 0xffffffffffffffffULL; - for(unsigned int p=0;preceived(now); -#ifdef ZT_ENABLE_CLUSTER - slot->setClusterSuboptimal(suboptimalPath); -#endif - _numPaths = np; + slot = oldestPath; } + _paths[slot].path = path; + _paths[slot].lastReceive = now; #ifdef ZT_ENABLE_CLUSTER + _paths[slot].clusterSuboptimal = suboptimalPath; + if (RR->cluster) RR->cluster->broadcastHavePeer(_id); #endif - } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str()); @@ -178,15 +169,15 @@ void Peer::received( if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); outp.armor(_key,true); - RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size()); + path->send(RR,outp.data(),outp.size(),now); } else { - sendHELLO(localAddr,remoteAddr,now); + sendHELLO(path->localAddress(),path->address(),now); } } } } else if (trustEstablished) { - _pushDirectPaths(localAddr,remoteAddr,now); + _pushDirectPaths(path,now); } if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) { @@ -197,7 +188,96 @@ void Peer::received( } } -void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl) +bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const +{ + Mutex::Lock _l(_paths_m); + for(unsigned int p=0;p<_numPaths;++p) { + if ( (_paths[p].path->address() == addr) && (_paths[p].path->alive(now)) ) + return true; + } + return false; +} + +void Peer::makeExclusive(const InetAddress &addr) +{ + Mutex::Lock _l(_paths_m); + + bool have = false; + for(unsigned int p=0;p<_numPaths;++p) { + if (_paths[p].path->address() == addr) { + have = true; + break; + } + } + + if (have) { + unsigned int np = _numPaths; + unsigned int x = 0; + unsigned int y = 0; + while (x < np) { + if ((_paths[x].path->address().ss_family != addr.ss_family)||(_paths[x].path->address() == addr)) { + if (y != x) { + _paths[y].path = _paths[x].path; + _paths[y].lastReceive = _paths[x].lastReceive; + #ifdef ZT_ENABLE_CLUSTER + _paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal; + #endif + } + ++y; + } + ++x; + } + _numPaths = y; + } +} + +bool Peer::send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) +{ + Mutex::Lock _l(_paths_m); + + int bestp = -1; + uint64_t best = 0ULL; + for(unsigned int p=0;p<_numPaths;++p) { + if (_paths[p].path->alive(now)||(forceEvenIfDead)) { + const uint64_t s = _paths[p].path->score(); + if (s >= best) { + best = s; + bestp = (int)p; + } + } + } + + if (bestp >= 0) { + return _paths[bestp].path->send(RR,data,len,now); + } else { + return false; + } +} + +SharedPtr Peer::getBestPath(uint64_t now,bool forceEvenIfDead) +{ + Mutex::Lock _l(_paths_m); + + int bestp = -1; + uint64_t best = 0ULL; + for(unsigned int p=0;p<_numPaths;++p) { + if (_paths[p].path->alive(now)||(forceEvenIfDead)) { + const uint64_t s = _paths[p].path->score(); + if (s >= best) { + best = s; + bestp = (int)p; + } + } + } + + if (bestp >= 0) { + return _paths[bestp].path; + } else { + return SharedPtr(); + } +} + +void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); outp.append((unsigned char)ZT_PROTO_VERSION); @@ -209,51 +289,56 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u atAddress.serialize(outp); outp.append((uint64_t)RR->topology->worldId()); outp.append((uint64_t)RR->topology->worldTimestamp()); - outp.armor(_key,false); // HELLO is sent in the clear - RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size(),ttl); + RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) { - Path *p = (Path *)0; - - if (inetAddressFamily != 0) { - p = _getBestPath(now,inetAddressFamily); - } else { - p = _getBestPath(now); - } - - if (p) { - if ((now - p->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) { - //TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived()); - sendHELLO(p->localAddress(),p->address(),now); - p->sent(now); - p->pinged(now); - } else if ((now - std::max(p->lastSend(),p->lastKeepalive())) >= ZT_NAT_KEEPALIVE_DELAY) { - //TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived()); + bool somethingAlive = false; + Mutex::Lock _l(_paths_m); + for(unsigned int p=0;p<_numPaths;++p) { + if ((now - _paths[p].lastReceive) >= ZT_PEER_PING_PERIOD) { + sendHELLO(_paths[p].path->localAddress(),_paths[p].path->address(),now); + } else if (_paths[p].path->needsHeartbeat(now)) { _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads - RR->node->putPacket(p->localAddress(),p->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf)); - p->sentKeepalive(now); + _paths[p].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now); } - return true; + somethingAlive |= _paths[p].path->alive(now); } + return somethingAlive; +} +bool Peer::hasActiveDirectPath(uint64_t now) const +{ + Mutex::Lock _l(_paths_m); + for(unsigned int p=0;p<_numPaths;++p) { + if (_paths[p].path->alive(now)) + return true; + } return false; } bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) { + Mutex::Lock _l(_paths_m); unsigned int np = _numPaths; unsigned int x = 0; unsigned int y = 0; while (x < np) { - if (_paths[x].address().ipScope() == scope) { + if (_paths[x].path->address().ipScope() == scope) { // Resetting a path means sending a HELLO and then forgetting it. If we // get OK(HELLO) then it will be re-learned. - sendHELLO(_paths[x].localAddress(),_paths[x].address(),now); + sendHELLO(_paths[x].path->localAddress(),_paths[x].path->address(),now); } else { - _paths[y++] = _paths[x]; + if (x != y) { + _paths[y].path = _paths[x].path; + _paths[y].lastReceive = _paths[x].lastReceive; +#ifdef ZT_ENABLE_CLUSTER + _paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal; +#endif + } + ++y; } ++x; } @@ -263,114 +348,55 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const { - uint64_t bestV4 = 0,bestV6 = 0; - for(unsigned int p=0,np=_numPaths;p= bestV4) { - bestV4 = lr; - v4 = _paths[p].address(); - } - } else if (_paths[p].address().isV6()) { - if (lr >= bestV6) { - bestV6 = lr; - v6 = _paths[p].address(); - } - } + Mutex::Lock _l(_paths_m); + + int bestp4 = -1,bestp6 = -1; + uint64_t best4 = 0ULL,best6 = 0ULL; + for(unsigned int p=0;p<_numPaths;++p) { + if (_paths[p].path->address().ss_family == AF_INET) { + const uint64_t s = _paths[p].path->score(); + if (s >= best4) { + best4 = s; + bestp4 = (int)p; + } + } else if (_paths[p].path->address().ss_family == AF_INET6) { + const uint64_t s = _paths[p].path->score(); + if (s >= best6) { + best6 = s; + bestp6 = (int)p; } } } + + if (bestp4 >= 0) + v4 = _paths[bestp4].path->address(); + if (bestp6 >= 0) + v6 = _paths[bestp6].path->address(); } void Peer::clean(uint64_t now) { + Mutex::Lock _l(_paths_m); unsigned int np = _numPaths; unsigned int x = 0; unsigned int y = 0; while (x < np) { - if (_paths[x].active(now)) - _paths[y++] = _paths[x]; + if ((now - _paths[x].lastReceive) <= ZT_PEER_PATH_EXPIRATION) { + if (y != x) { + _paths[y].path = _paths[x].path; + _paths[y].lastReceive = _paths[x].lastReceive; +#ifdef ZT_ENABLE_CLUSTER + _paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal; +#endif + } + ++y; + } ++x; } _numPaths = y; } -void Peer::_doDeadPathDetection(Path &p,const uint64_t now) -{ - /* Dead path detection: if we have sent something to this peer and have not - * yet received a reply, double check this path. The majority of outbound - * packets including Ethernet frames do generate some kind of reply either - * immediately or at some point in the near future. This will occasionally - * (every NO_ANSWER_TIMEOUT ms) check paths unnecessarily if traffic that - * does not generate a response is being sent such as multicast announcements - * or frames belonging to unidirectional UDP protocols, but the cost is very - * tiny and the benefit in reliability is very large. This takes care of many - * failure modes including crap NATs that forget links and spurious changes - * to physical network topology that cannot be otherwise detected. - * - * Each time we do this we increment a probation counter in the path. This - * counter is reset on any packet receive over this path. If it reaches the - * MAX_PROBATION threshold the path is considred dead. */ - - if ( - (p.lastSend() > p.lastReceived()) && - ((p.lastSend() - p.lastReceived()) >= ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT) && - ((now - p.lastPing()) >= ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT) && - (!p.isClusterSuboptimal()) && - (!RR->topology->amRoot()) - ) { - TRACE("%s(%s) does not seem to be answering in a timely manner, checking if dead (probation == %u)",_id.address().toString().c_str(),p.address().toString().c_str(),p.probation()); - - if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); - outp.armor(_key,true); - p.send(RR,outp.data(),outp.size(),now); - p.pinged(now); - } else { - sendHELLO(p.localAddress(),p.address(),now); - p.sent(now); - p.pinged(now); - } - - p.increaseProbation(); - } -} - -Path *Peer::_getBestPath(const uint64_t now) -{ - Path *bestPath = (Path *)0; - uint64_t bestPathScore = 0; - for(unsigned int i=0;i<_numPaths;++i) { - const uint64_t score = _paths[i].score(); - if ((score >= bestPathScore)&&(_paths[i].active(now))) { - bestPathScore = score; - bestPath = &(_paths[i]); - } - } - if (bestPath) - _doDeadPathDetection(*bestPath,now); - return bestPath; -} - -Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily) -{ - Path *bestPath = (Path *)0; - uint64_t bestPathScore = 0; - for(unsigned int i=0;i<_numPaths;++i) { - const uint64_t score = _paths[i].score(); - if (((int)_paths[i].address().ss_family == inetAddressFamily)&&(score >= bestPathScore)&&(_paths[i].active(now))) { - bestPathScore = score; - bestPath = &(_paths[i]); - } - } - if (bestPath) - _doDeadPathDetection(*bestPath,now); - return bestPath; -} - -bool Peer::_pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now) +bool Peer::_pushDirectPaths(const SharedPtr &path,uint64_t now) { #ifdef ZT_ENABLE_CLUSTER // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection @@ -445,7 +471,7 @@ bool Peer::_pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAd if (count) { outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); outp.armor(_key,true); - RR->node->putPacket(localAddr,toAddress,outp.data(),outp.size(),0); + path->send(RR,outp.data(),outp.size(),now); } } diff --git a/node/Peer.hpp b/node/Peer.hpp index a6940737..ff32184f 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -96,9 +96,7 @@ public: * This is called by the decode pipe when a packet is proven to be authentic * and appears to be valid. * - * @param RR Runtime environment - * @param localAddr Local address - * @param remoteAddr Internet address of sender + * @param path Path over which packet was received * @param hops ZeroTier (not IP) hops * @param packetId Packet ID * @param verb Packet verb @@ -107,8 +105,7 @@ public: * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established */ void received( - const InetAddress &localAddr, - const InetAddress &remoteAddr, + const SharedPtr &path, unsigned int hops, uint64_t packetId, Packet::Verb verb, @@ -116,43 +113,19 @@ public: Packet::Verb inReVerb, const bool trustEstablished); - /** - * Get the current best direct path to this peer - * - * @param now Current time - * @return Best path or NULL if there are no active direct paths - */ - inline Path *getBestPath(uint64_t now) { return _getBestPath(now); } - /** * @param now Current time * @param addr Remote address * @return True if we have an active path to this destination */ - inline bool hasActivePathTo(uint64_t now,const InetAddress &addr) const - { - for(unsigned int p=0;p<_numPaths;++p) { - if ((_paths[p].active(now))&&(_paths[p].address() == addr)) - return true; - } - return false; - } + bool hasActivePathTo(uint64_t now,const InetAddress &addr) const; /** - * Set all paths in the same ss_family that are not this one to cluster suboptimal - * - * Addresses in other families are not affected. + * If we have a confirmed path to this address, forget all others within the same address family * * @param addr Address to make exclusive */ - inline void setClusterOptimalPathForAddressFamily(const InetAddress &addr) - { - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].address().ss_family == addr.ss_family) { - _paths[p].setClusterSuboptimal(_paths[p].address() != addr); - } - } - } + void makeExclusive(const InetAddress &addr); /** * Send via best path @@ -160,30 +133,30 @@ public: * @param data Packet data * @param len Packet length * @param now Current time - * @return Path used on success or NULL on failure + * @param forceEvenIfDead If true, send even if the path is not 'alive' + * @return True if we actually sent something */ - inline Path *send(const void *data,unsigned int len,uint64_t now) - { - Path *const bestPath = getBestPath(now); - if (bestPath) { - if (bestPath->send(RR,data,len,now)) - return bestPath; - } - return (Path *)0; - } + bool send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); + + /** + * Get the best current direct path + * + * @param now Current time + * @param forceEvenIfDead If true, pick even if path is not alive + * @return Best current path or NULL if none + */ + SharedPtr getBestPath(uint64_t now,bool forceEvenIfDead); /** * Send a HELLO to this peer at a specified physical address * - * This does not update any statistics. It's used to send initial HELLOs - * for NAT traversal and path verification. + * No statistics or sent times are updated here. * * @param localAddr Local address * @param atAddress Destination address * @param now Current time - * @param ttl Desired IP TTL (default: 0 to leave alone) */ - void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl = 0); + void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now); /** * Send pings or keepalives depending on configured timeouts @@ -194,14 +167,49 @@ public: */ bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); + /** + * @param now Current time + * @return True if this peer has at least one active direct path + */ + bool hasActiveDirectPath(uint64_t now) const; + + /** + * Reset paths within a given scope + * + * @param scope IP scope of paths to reset + * @param now Current time + * @return True if at least one path was forgotten + */ + bool resetWithinScope(InetAddress::IpScope scope,uint64_t now); + + /** + * Get most recently active path addresses for IPv4 and/or IPv6 + * + * Note that v4 and v6 are not modified if they are not found, so + * initialize these to a NULL address to be able to check. + * + * @param now Current time + * @param v4 Result parameter to receive active IPv4 address, if any + * @param v6 Result parameter to receive active IPv6 address, if any + */ + void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; + + /** + * Perform periodic cleaning operations + * + * @param now Current time + */ + void clean(uint64_t now); + /** * @return All known direct paths to this peer (active or inactive) */ - inline std::vector paths() const + inline std::vector< SharedPtr > paths() const { - std::vector pp; + std::vector< SharedPtr > pp; + Mutex::Lock _l(_paths_m); for(unsigned int p=0,np=_numPaths;palive(now)) && (!_paths[p].clusterSuboptimal) ) return true; } return false; } #endif - /** - * Reset paths within a given scope - * - * @param scope IP scope of paths to reset - * @param now Current time - * @return True if at least one path was forgotten - */ - bool resetWithinScope(InetAddress::IpScope scope,uint64_t now); - /** * @return 256-bit secret symmetric encryption key */ @@ -335,25 +321,6 @@ public: inline bool remoteVersionKnown() const throw() { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); } - /** - * Get most recently active path addresses for IPv4 and/or IPv6 - * - * Note that v4 and v6 are not modified if they are not found, so - * initialize these to a NULL address to be able to check. - * - * @param now Current time - * @param v4 Result parameter to receive active IPv4 address, if any - * @param v6 Result parameter to receive active IPv6 address, if any - */ - void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; - - /** - * Perform periodic cleaning operations - * - * @param now Current time - */ - void clean(uint64_t now); - /** * Update direct path push stats and return true if we should respond * @@ -395,10 +362,7 @@ public: } private: - void _doDeadPathDetection(Path &p,const uint64_t now); - Path *_getBestPath(const uint64_t now); - Path *_getBestPath(const uint64_t now,int inetAddressFamily); - bool _pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now); + bool _pushDirectPaths(const SharedPtr &path,uint64_t now); unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; @@ -410,13 +374,19 @@ private: uint64_t _lastAnnouncedTo; uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; - uint64_t _lastPathSort; uint16_t _vProto; uint16_t _vMajor; uint16_t _vMinor; uint16_t _vRevision; Identity _id; - Path _paths[ZT_MAX_PEER_NETWORK_PATHS]; + struct { + SharedPtr path; + uint64_t lastReceive; +#ifdef ZT_ENABLE_CLUSTER + bool clusterSuboptimal; +#endif + } _paths[ZT_MAX_PEER_NETWORK_PATHS]; + Mutex _paths_m; unsigned int _numPaths; unsigned int _latency; unsigned int _directPathPushCutoffCount; diff --git a/node/Switch.cpp b/node/Switch.cpp index dc238607..ab07d353 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -112,7 +112,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Note: we don't bother initiating NAT-t for fragments, since heads will set that off. // It wouldn't hurt anything, just redundant and unnecessary. SharedPtr relayTo = RR->topology->getPeer(destination); - if ((!relayTo)||(!relayTo->send(fragment.data(),fragment.size(),now))) { + if ((!relayTo)||(!relayTo->send(fragment.data(),fragment.size(),now,true))) { #ifdef ZT_ENABLE_CLUSTER if (RR->cluster) { RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false); @@ -123,7 +123,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Don't know peer or no direct path -- so relay via root server relayTo = RR->topology->getBestRoot(); if (relayTo) - relayTo->send(fragment.data(),fragment.size(),now); + relayTo->send(fragment.data(),fragment.size(),now,true); } } else { TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str()); @@ -210,7 +210,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from packet.incrementHops(); SharedPtr relayTo = RR->topology->getPeer(destination); - if ((relayTo)&&((relayTo->send(packet.data(),packet.size(),now)))) { + if ((relayTo)&&((relayTo->send(packet.data(),packet.size(),now,true)))) { Mutex::Lock _l(_lastUniteAttempt_m); uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) { @@ -234,7 +234,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif relayTo = RR->topology->getBestRoot(&source,1,true); if (relayTo) - relayTo->send(packet.data(),packet.size(),now); + relayTo->send(packet.data(),packet.size(),now,true); } } else { TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet.source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str()); @@ -251,7 +251,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from rq->timestamp = now; rq->packetId = packetId; - rq->frag0.init(data,len,localAddr,fromAddr,now); + rq->frag0.init(data,len,path,now); rq->totalFragments = 0; rq->haveFragments = 1; rq->complete = false; @@ -607,7 +607,7 @@ bool Switch::unite(const Address &p1,const Address &p2) outp.append(cg.first.rawIpData(),4); } outp.armor(p1p->key(),true); - p1p->send(outp.data(),outp.size(),now); + p1p->send(outp.data(),outp.size(),now,true); } else { // Tell p2 where to find p1. Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS); @@ -622,7 +622,7 @@ bool Switch::unite(const Address &p1,const Address &p2) outp.append(cg.second.rawIpData(),4); } outp.armor(p2p->key(),true); - p2p->send(outp.data(),outp.size(),now); + p2p->send(outp.data(),outp.size(),now,true); } ++alt; // counts up and also flips LSB } @@ -739,7 +739,7 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); outp.armor(root->key(),true); - if (root->send(outp.data(),outp.size(),RR->node->now())) + if (root->send(outp.data(),outp.size(),RR->node->now(),true)) return root->address(); } return Address(); @@ -752,12 +752,10 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) if (peer) { const uint64_t now = RR->node->now(); - Path *viaPath = peer->getBestPath(now); - SharedPtr relay; - + SharedPtr viaPath(peer->getBestPath(now,false)); if (!viaPath) { - relay = RR->topology->getBestRoot(); - if ( (!relay) || (!(viaPath = relay->getBestPath(now))) ) + SharedPtr relay(RR->topology->getBestRoot()); + if ( (!relay) || (!(viaPath = relay->getBestPath(now,true))) ) return false; } -- cgit v1.2.3 From d1101441b3d43ee69c1b661cc5f777a09fd10fca Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 11:54:59 -0700 Subject: Tweak some timings. --- node/Constants.hpp | 6 +++--- node/Peer.cpp | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 6d6f44e0..b783e2c2 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -244,17 +244,17 @@ * This is also how often pings will be retried to upstream peers (relays, roots) * constantly until something is heard. */ -#define ZT_PING_CHECK_INVERVAL 8000 +#define ZT_PING_CHECK_INVERVAL 9000 /** * How frequently to send heartbeats over in-use paths */ -#define ZT_PATH_HEARTBEAT_PERIOD 18000 +#define ZT_PATH_HEARTBEAT_PERIOD 15000 /** * Paths are considered inactive if they have not received traffic in this long */ -#define ZT_PATH_ALIVE_TIMEOUT ((ZT_PATH_HEARTBEAT_PERIOD * 2) + 2000) +#define ZT_PATH_ALIVE_TIMEOUT 35000 /** * Delay between full-fledge pings of directly connected peers diff --git a/node/Peer.cpp b/node/Peer.cpp index 251c5a5f..9b5d84fc 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -27,8 +27,6 @@ #include "Cluster.hpp" #include "Packet.hpp" -#define ZT_PEER_PATH_SORT_INTERVAL 5000 - namespace ZeroTier { // Used to send varying values for NAT keepalive -- cgit v1.2.3 From 4931e449989f74b9518d15bc69521fdbefb313e7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 12:34:02 -0700 Subject: Implement "weak pointer" behavior on Topology Path canonicalization hash table. --- node/AtomicCounter.hpp | 51 ++++---------------------------------------------- node/SharedPtr.hpp | 28 +++++++++++++++++++++++++-- node/Topology.cpp | 27 ++++++++++++++++++-------- 3 files changed, 49 insertions(+), 57 deletions(-) (limited to 'node') diff --git a/node/AtomicCounter.hpp b/node/AtomicCounter.hpp index b4993771..a0f29baa 100644 --- a/node/AtomicCounter.hpp +++ b/node/AtomicCounter.hpp @@ -20,11 +20,9 @@ #define ZT_ATOMICCOUNTER_HPP #include "Constants.hpp" -#include "Mutex.hpp" #include "NonCopyable.hpp" -#ifdef __WINDOWS__ -// will replace this whole class eventually once it's ubiquitous +#ifndef __GNUC__ #include #endif @@ -36,75 +34,34 @@ namespace ZeroTier { class AtomicCounter : NonCopyable { public: - /** - * Initialize counter at zero - */ AtomicCounter() - throw() { _v = 0; } - inline operator int() const - throw() - { -#ifdef __GNUC__ - return __sync_or_and_fetch(const_cast (&_v),0); -#else -#ifdef __WINDOWS__ - return (int)_v; -#else - _l.lock(); - int v = _v; - _l.unlock(); - return v; -#endif -#endif - } - inline int operator++() - throw() { #ifdef __GNUC__ return __sync_add_and_fetch(&_v,1); #else -#ifdef __WINDOWS__ return ++_v; -#else - _l.lock(); - int v = ++_v; - _l.unlock(); - return v; -#endif #endif } inline int operator--() - throw() { #ifdef __GNUC__ return __sync_sub_and_fetch(&_v,1); #else -#ifdef __WINDOWS__ return --_v; -#else - _l.lock(); - int v = --_v; - _l.unlock(); - return v; -#endif #endif } private: -#ifdef __WINDOWS__ - std::atomic_int _v; -#else +#ifdef __GNUC__ int _v; -#ifndef __GNUC__ -#warning Neither __WINDOWS__ nor __GNUC__ so AtomicCounter using Mutex - Mutex _l; -#endif +#else + std::atomic_int _v; #endif }; diff --git a/node/SharedPtr.hpp b/node/SharedPtr.hpp index 3ff5ed18..1dd3b43d 100644 --- a/node/SharedPtr.hpp +++ b/node/SharedPtr.hpp @@ -119,15 +119,39 @@ public: inline T *ptr() const throw() { return _ptr; } /** - * Set this pointer to null + * Set this pointer to NULL */ inline void zero() { if (_ptr) { if (--_ptr->__refCount <= 0) delete _ptr; + _ptr = (T *)0; + } + } + + /** + * Set this pointer to NULL if this is the only pointer holding the object + * + * @return True if object was deleted and SharedPtr is now NULL (or was already NULL) + */ + inline bool reclaimIfWeak() + { + if (_ptr) { + if (++_ptr->__refCount <= 2) { + if (--_ptr->__refCount <= 1) { + delete _ptr; + _ptr = (T *)0; + return true; + } else { + return false; + } + } else { + return false; + } + } else { + return true; } - _ptr = (T *)0; } inline bool operator==(const SharedPtr &sp) const throw() { return (_ptr == sp._ptr); } diff --git a/node/Topology.cpp b/node/Topology.cpp index 6e0fe90c..c6d46dc5 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -251,14 +251,25 @@ bool Topology::worldUpdateIfValid(const World &newWorld) void Topology::clean(uint64_t now) { Mutex::Lock _l(_lock); - Hashtable< Address,SharedPtr >::Iterator i(_peers); - Address *a = (Address *)0; - SharedPtr *p = (SharedPtr *)0; - while (i.next(a,p)) { - if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) { - _peers.erase(*a); - } else { - (*p)->clean(now); + { + Hashtable< Address,SharedPtr >::Iterator i(_peers); + Address *a = (Address *)0; + SharedPtr *p = (SharedPtr *)0; + while (i.next(a,p)) { + if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) { + _peers.erase(*a); + } else { + (*p)->clean(now); + } + } + } + { + Hashtable< Path::HashKey,SharedPtr >::Iterator i(_paths); + Path::HashKey *k = (Path::HashKey *)0; + SharedPtr *p = (SharedPtr *)0; + while (i.next(k,p)) { + if (p->reclaimIfWeak()) + _paths.erase(*k); } } } -- cgit v1.2.3 From 4f8253dcdb9e6ff2ae18639556811d13729fae2b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 13:33:56 -0700 Subject: Tweaks to path handling... --- node/IncomingPacket.cpp | 4 ++-- node/Node.cpp | 2 +- node/Peer.cpp | 39 ++++++++++++++++++++++++--------------- node/Peer.hpp | 7 +++---- node/Switch.cpp | 21 +++++++++++---------- 5 files changed, 41 insertions(+), 32 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index fafd5679..7ba34566 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1163,8 +1163,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt remainingHopsPtr += ZT_ADDRESS_LENGTH; SharedPtr nhp(RR->topology->getPeer(nextHop[h])); if (nhp) { - SharedPtr nhbp(nhp->getBestPath(now,false)); - if (nhbp) + SharedPtr nhbp(nhp->getBestPath(now)); + if ((nhbp)&&(nhbp->alive(now))) nextHopBestPathAddress[h] = nhbp->address(); } } diff --git a/node/Node.cpp b/node/Node.cpp index 39e24325..d2840bd0 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -416,7 +416,7 @@ ZT_PeerList *Node::peers() const p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF; std::vector< SharedPtr > paths(pi->second->paths()); - SharedPtr bestp(pi->second->getBestPath(_now,true)); + SharedPtr bestp(pi->second->getBestPath(_now)); p->pathCount = 0; for(std::vector< SharedPtr >::iterator path(paths.begin());path!=paths.end();++path) { memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); diff --git a/node/Peer.cpp b/node/Peer.cpp index 9b5d84fc..a23d0822 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -229,7 +229,7 @@ void Peer::makeExclusive(const InetAddress &addr) } } -bool Peer::send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) +bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) { Mutex::Lock _l(_paths_m); @@ -252,19 +252,17 @@ bool Peer::send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfD } } -SharedPtr Peer::getBestPath(uint64_t now,bool forceEvenIfDead) +SharedPtr Peer::getBestPath(uint64_t now) { Mutex::Lock _l(_paths_m); int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->alive(now)||(forceEvenIfDead)) { - const uint64_t s = _paths[p].path->score(); - if (s >= best) { - best = s; - bestp = (int)p; - } + const uint64_t s = _paths[p].path->score(); + if (s >= best) { + best = s; + bestp = (int)p; } } @@ -293,18 +291,29 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) { - bool somethingAlive = false; Mutex::Lock _l(_paths_m); + + int bestp = -1; + uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if ((now - _paths[p].lastReceive) >= ZT_PEER_PING_PERIOD) { - sendHELLO(_paths[p].path->localAddress(),_paths[p].path->address(),now); - } else if (_paths[p].path->needsHeartbeat(now)) { + const uint64_t s = _paths[p].path->score(); + if (s >= best) { + best = s; + bestp = (int)p; + } + } + + if (bestp >= 0) { + if ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) { + sendHELLO(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); + } else if (_paths[bestp].path->needsHeartbeat(now)) { _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads - _paths[p].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now); + _paths[bestp].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now); } - somethingAlive |= _paths[p].path->alive(now); + return true; + } else { + return false; } - return somethingAlive; } bool Peer::hasActiveDirectPath(uint64_t now) const diff --git a/node/Peer.hpp b/node/Peer.hpp index ff32184f..87aea486 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -128,7 +128,7 @@ public: void makeExclusive(const InetAddress &addr); /** - * Send via best path + * Send via best direct path * * @param data Packet data * @param len Packet length @@ -136,16 +136,15 @@ public: * @param forceEvenIfDead If true, send even if the path is not 'alive' * @return True if we actually sent something */ - bool send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); + bool sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); /** * Get the best current direct path * * @param now Current time - * @param forceEvenIfDead If true, pick even if path is not alive * @return Best current path or NULL if none */ - SharedPtr getBestPath(uint64_t now,bool forceEvenIfDead); + SharedPtr getBestPath(uint64_t now); /** * Send a HELLO to this peer at a specified physical address diff --git a/node/Switch.cpp b/node/Switch.cpp index ab07d353..125c4b69 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -75,6 +75,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from SharedPtr path(RR->topology->getPath(localAddr,fromAddr)); path->received(now); + printf("<< %s %u\n",fromAddr.toString().c_str(),len); if (len == 13) { /* LEGACY: before VERB_PUSH_DIRECT_PATHS, peers used broadcast @@ -112,7 +113,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Note: we don't bother initiating NAT-t for fragments, since heads will set that off. // It wouldn't hurt anything, just redundant and unnecessary. SharedPtr relayTo = RR->topology->getPeer(destination); - if ((!relayTo)||(!relayTo->send(fragment.data(),fragment.size(),now,true))) { + if ((!relayTo)||(!relayTo->sendDirect(fragment.data(),fragment.size(),now,false))) { #ifdef ZT_ENABLE_CLUSTER if (RR->cluster) { RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false); @@ -123,7 +124,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Don't know peer or no direct path -- so relay via root server relayTo = RR->topology->getBestRoot(); if (relayTo) - relayTo->send(fragment.data(),fragment.size(),now,true); + relayTo->sendDirect(fragment.data(),fragment.size(),now,true); } } else { TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str()); @@ -210,7 +211,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from packet.incrementHops(); SharedPtr relayTo = RR->topology->getPeer(destination); - if ((relayTo)&&((relayTo->send(packet.data(),packet.size(),now,true)))) { + if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { Mutex::Lock _l(_lastUniteAttempt_m); uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) { @@ -234,7 +235,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif relayTo = RR->topology->getBestRoot(&source,1,true); if (relayTo) - relayTo->send(packet.data(),packet.size(),now,true); + relayTo->sendDirect(packet.data(),packet.size(),now,true); } } else { TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet.source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str()); @@ -607,7 +608,7 @@ bool Switch::unite(const Address &p1,const Address &p2) outp.append(cg.first.rawIpData(),4); } outp.armor(p1p->key(),true); - p1p->send(outp.data(),outp.size(),now,true); + p1p->sendDirect(outp.data(),outp.size(),now,true); } else { // Tell p2 where to find p1. Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS); @@ -622,7 +623,7 @@ bool Switch::unite(const Address &p1,const Address &p2) outp.append(cg.second.rawIpData(),4); } outp.armor(p2p->key(),true); - p2p->send(outp.data(),outp.size(),now,true); + p2p->sendDirect(outp.data(),outp.size(),now,true); } ++alt; // counts up and also flips LSB } @@ -739,7 +740,7 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); outp.armor(root->key(),true); - if (root->send(outp.data(),outp.size(),RR->node->now(),true)) + if (root->sendDirect(outp.data(),outp.size(),RR->node->now(),true)) return root->address(); } return Address(); @@ -752,10 +753,10 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) if (peer) { const uint64_t now = RR->node->now(); - SharedPtr viaPath(peer->getBestPath(now,false)); - if (!viaPath) { + SharedPtr viaPath(peer->getBestPath(now)); + if ( (!viaPath) || ((!viaPath->alive(now))&&(!RR->topology->isRoot(peer->identity()))) ) { SharedPtr relay(RR->topology->getBestRoot()); - if ( (!relay) || (!(viaPath = relay->getBestPath(now,true))) ) + if ( (!relay) || (!(viaPath = relay->getBestPath(now))) ) return false; } -- cgit v1.2.3 From 412979ba8f0e1776648727765126c210683e038d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 13:55:33 -0700 Subject: Attempt to reactivate dead paths. --- node/Peer.hpp | 2 +- node/Switch.cpp | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Peer.hpp b/node/Peer.hpp index 87aea486..294a4913 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -162,7 +162,7 @@ public: * * @param now Current time * @param inetAddressFamily Keep this address family alive, or 0 to simply pick current best ignoring family - * @return True if at least one direct path seems alive + * @return True if we have at least one direct path */ bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); diff --git a/node/Switch.cpp b/node/Switch.cpp index 125c4b69..472440d1 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -754,7 +754,15 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) const uint64_t now = RR->node->now(); SharedPtr viaPath(peer->getBestPath(now)); - if ( (!viaPath) || ((!viaPath->alive(now))&&(!RR->topology->isRoot(peer->identity()))) ) { + if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) { + if ((now - viaPath->lastOut()) > 5000) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ECHO); + outp.armor(peer->key(),true); + viaPath->send(RR,outp.data(),outp.size(),now); + } + viaPath.zero(); + } + if (!viaPath) { SharedPtr relay(RR->topology->getBestRoot()); if ( (!relay) || (!(viaPath = relay->getBestPath(now))) ) return false; -- cgit v1.2.3 From 4992ac2d9f568a08ecd04b316a926c9d320750df Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 14:20:55 -0700 Subject: Cluster sub-optimal is in fact necessary... --- node/IncomingPacket.cpp | 4 ++-- node/Path.hpp | 8 -------- node/Peer.cpp | 40 +++++++++++++--------------------------- node/Peer.hpp | 11 +++++++---- 4 files changed, 22 insertions(+), 41 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 7ba34566..a84b2beb 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1042,7 +1042,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha bool redundant = false; if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->makeExclusive(a); + peer->setClusterOptimal(a); } else { redundant = peer->hasActivePathTo(now,a); } @@ -1061,7 +1061,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha bool redundant = false; if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->makeExclusive(a); + peer->setClusterOptimal(a); } else { redundant = peer->hasActivePathTo(now,a); } diff --git a/node/Path.hpp b/node/Path.hpp index 68a630c3..d0e1d737 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -165,14 +165,6 @@ public: return ( ((unsigned int)_ipScope << 1) | (unsigned int)(_addr.ss_family == AF_INET6) ); } - /** - * @return This path's overall quality score (higher is better) - */ - inline uint64_t score() const - { - return (_lastIn + (preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK))); - } - /** * Check whether this address is valid for a ZeroTier path * diff --git a/node/Peer.cpp b/node/Peer.cpp index a23d0822..ecf2a870 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -156,9 +156,10 @@ void Peer::received( _paths[slot].lastReceive = now; #ifdef ZT_ENABLE_CLUSTER _paths[slot].clusterSuboptimal = suboptimalPath; - if (RR->cluster) RR->cluster->broadcastHavePeer(_id); +#else + _paths[slot].clusterSuboptimal = false; #endif } else { @@ -196,36 +197,21 @@ bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const return false; } -void Peer::makeExclusive(const InetAddress &addr) +void Peer::setClusterOptimal(const InetAddress &addr) { Mutex::Lock _l(_paths_m); - bool have = false; + int have = -1; for(unsigned int p=0;p<_numPaths;++p) { if (_paths[p].path->address() == addr) { - have = true; + have = (int)p; break; } } - if (have) { - unsigned int np = _numPaths; - unsigned int x = 0; - unsigned int y = 0; - while (x < np) { - if ((_paths[x].path->address().ss_family != addr.ss_family)||(_paths[x].path->address() == addr)) { - if (y != x) { - _paths[y].path = _paths[x].path; - _paths[y].lastReceive = _paths[x].lastReceive; - #ifdef ZT_ENABLE_CLUSTER - _paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal; - #endif - } - ++y; - } - ++x; - } - _numPaths = y; + if (have >= 0) { + for(unsigned int p=0;p<_numPaths;++p) + _paths[p].clusterSuboptimal = (p != have); } } @@ -237,7 +223,7 @@ bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceE uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { if (_paths[p].path->alive(now)||(forceEvenIfDead)) { - const uint64_t s = _paths[p].path->score(); + const uint64_t s = _pathScore(p); if (s >= best) { best = s; bestp = (int)p; @@ -259,7 +245,7 @@ SharedPtr Peer::getBestPath(uint64_t now) int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _paths[p].path->score(); + const uint64_t s = _pathScore(p); if (s >= best) { best = s; bestp = (int)p; @@ -296,7 +282,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _paths[p].path->score(); + const uint64_t s = _pathScore(p); if (s >= best) { best = s; bestp = (int)p; @@ -361,13 +347,13 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) uint64_t best4 = 0ULL,best6 = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { if (_paths[p].path->address().ss_family == AF_INET) { - const uint64_t s = _paths[p].path->score(); + const uint64_t s = _pathScore(p); if (s >= best4) { best4 = s; bestp4 = (int)p; } } else if (_paths[p].path->address().ss_family == AF_INET6) { - const uint64_t s = _paths[p].path->score(); + const uint64_t s = _pathScore(p); if (s >= best6) { best6 = s; bestp6 = (int)p; diff --git a/node/Peer.hpp b/node/Peer.hpp index 294a4913..efe34825 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -121,11 +121,11 @@ public: bool hasActivePathTo(uint64_t now,const InetAddress &addr) const; /** - * If we have a confirmed path to this address, forget all others within the same address family + * If we have a confirmed path to this address, mark others as cluster suboptimal * * @param addr Address to make exclusive */ - void makeExclusive(const InetAddress &addr); + void setClusterOptimal(const InetAddress &addr); /** * Send via best direct path @@ -363,6 +363,11 @@ public: private: bool _pushDirectPaths(const SharedPtr &path,uint64_t now); + inline uint64_t _pathScore(const unsigned int p) const + { + return ( (_paths[p].path->lastIn() + (_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK))) - ((ZT_PEER_PING_PERIOD * 10) * (uint64_t)_paths[p].clusterSuboptimal) ); + } + unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; const RuntimeEnvironment *RR; @@ -381,9 +386,7 @@ private: struct { SharedPtr path; uint64_t lastReceive; -#ifdef ZT_ENABLE_CLUSTER bool clusterSuboptimal; -#endif } _paths[ZT_MAX_PEER_NETWORK_PATHS]; Mutex _paths_m; unsigned int _numPaths; -- cgit v1.2.3 From 01aa469591019d0412cd6714312467823aaacea9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 2 Sep 2016 14:26:04 -0700 Subject: Remove debug line. --- node/Switch.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 472440d1..28a2564b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -75,7 +75,6 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from SharedPtr path(RR->topology->getPath(localAddr,fromAddr)); path->received(now); - printf("<< %s %u\n",fromAddr.toString().c_str(),len); if (len == 13) { /* LEGACY: before VERB_PUSH_DIRECT_PATHS, peers used broadcast -- cgit v1.2.3 From eebcf08084097fc7cd8703a11686e66157fa8efa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 3 Sep 2016 15:39:05 -0700 Subject: Tweaks to new Path code for dual-stack operation, and other fixes. --- include/ZeroTierOne.h | 5 ----- node/Constants.hpp | 2 +- node/Network.cpp | 2 +- node/Node.cpp | 3 +-- node/Peer.cpp | 58 +++++++++++++++++++++++++----------------------- node/Peer.hpp | 16 ++++++++----- one.cpp | 10 ++++----- service/ControlPlane.cpp | 4 ++-- 8 files changed, 50 insertions(+), 50 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index db0560a3..583abfe4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1053,11 +1053,6 @@ typedef struct */ uint64_t trustedPathId; - /** - * Is path active? - */ - int active; - /** * Is path preferred? */ diff --git a/node/Constants.hpp b/node/Constants.hpp index b783e2c2..ea4c434d 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -264,7 +264,7 @@ /** * Peers forget paths that have not spoken in this long */ -#define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 3) + 3000) +#define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) /** * Timeout for overall peer activity (measured from last receive) diff --git a/node/Network.cpp b/node/Network.cpp index 4dc8aa30..b18a3b22 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -36,7 +36,7 @@ #include "Peer.hpp" // Uncomment to make the rules engine dump trace info to stdout -#define ZT_RULES_ENGINE_DEBUGGING 1 +//#define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { diff --git a/node/Node.cpp b/node/Node.cpp index d2840bd0..a7d4cfa9 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -243,7 +243,7 @@ public: lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); } else if (p->activelyTransferringFrames(_now)) { // Normal nodes get their preferred link kept alive if the node has generated frame traffic recently - p->doPingAndKeepalive(_now,0); + p->doPingAndKeepalive(_now,-1); } } @@ -422,7 +422,6 @@ ZT_PeerList *Node::peers() const memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); p->paths[p->pathCount].lastSend = (*path)->lastOut(); p->paths[p->pathCount].lastReceive = (*path)->lastIn(); - p->paths[p->pathCount].active = (*path)->alive(_now) ? 1 : 0; p->paths[p->pathCount].preferred = (*path == bestp) ? 1 : 0; p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); ++p->pathCount; diff --git a/node/Peer.cpp b/node/Peer.cpp index ecf2a870..0cb55fd8 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -122,10 +122,11 @@ void Peer::received( { Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path == path) { // paths are canonicalized so pointer compare is good here + if (_paths[p].path->address() == path->address()) { _paths[p].lastReceive = now; + _paths[p].path = path; // local address may have changed! #ifdef ZT_ENABLE_CLUSTER - _paths[p].clusterSuboptimal = suboptimalPath; + _paths[p].clusterWeights = (unsigned int)(!suboptimalPath); #endif pathIsConfirmed = true; break; @@ -141,25 +142,26 @@ void Peer::received( if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) { slot = _numPaths++; } else { - uint64_t oldest = 0ULL; - unsigned int oldestPath = 0; + uint64_t worstScore = 0xffffffffffffffffULL; + unsigned int worstPath = ZT_MAX_PEER_NETWORK_PATHS-1; for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].lastReceive < oldest) { - oldest = _paths[p].lastReceive; - oldestPath = p; + const uint64_t s = _pathScore(p); + if (s < worstScore) { + worstScore = s; + worstPath = p; } } - slot = oldestPath; + slot = worstPath; } - _paths[slot].path = path; _paths[slot].lastReceive = now; + _paths[slot].path = path; #ifdef ZT_ENABLE_CLUSTER - _paths[slot].clusterSuboptimal = suboptimalPath; + _paths[slot].clusterWeights = (unsigned int)(!suboptimalPath); if (RR->cluster) RR->cluster->broadcastHavePeer(_id); #else - _paths[slot].clusterSuboptimal = false; + _paths[slot].clusterWeights = 1; #endif } else { @@ -201,17 +203,19 @@ void Peer::setClusterOptimal(const InetAddress &addr) { Mutex::Lock _l(_paths_m); - int have = -1; + int opt = -1; for(unsigned int p=0;p<_numPaths;++p) { if (_paths[p].path->address() == addr) { - have = (int)p; + opt = (int)p; break; } } - if (have >= 0) { - for(unsigned int p=0;p<_numPaths;++p) - _paths[p].clusterSuboptimal = (p != have); + if (opt >= 0) { // only change anything if we have the optimal path + for(unsigned int p=0;p<_numPaths;++p) { + if (_paths[p].path->address().ss_family == addr.ss_family) + _paths[p].clusterWeights = ((int)p == opt) ? 2 : 0; + } } } @@ -282,10 +286,12 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _pathScore(p); - if (s >= best) { - best = s; - bestp = (int)p; + if ((inetAddressFamily < 0)||(_paths[p].path->address().ss_family == inetAddressFamily)) { + const uint64_t s = _pathScore(p); + if (s >= best) { + best = s; + bestp = (int)p; + } } } @@ -325,11 +331,9 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) sendHELLO(_paths[x].path->localAddress(),_paths[x].path->address(),now); } else { if (x != y) { - _paths[y].path = _paths[x].path; _paths[y].lastReceive = _paths[x].lastReceive; -#ifdef ZT_ENABLE_CLUSTER - _paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal; -#endif + _paths[y].path = _paths[x].path; + _paths[y].clusterWeights = _paths[x].clusterWeights; } ++y; } @@ -376,11 +380,9 @@ void Peer::clean(uint64_t now) while (x < np) { if ((now - _paths[x].lastReceive) <= ZT_PEER_PATH_EXPIRATION) { if (y != x) { - _paths[y].path = _paths[x].path; _paths[y].lastReceive = _paths[x].lastReceive; -#ifdef ZT_ENABLE_CLUSTER - _paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal; -#endif + _paths[y].path = _paths[x].path; + _paths[y].clusterWeights = _paths[x].clusterWeights; } ++y; } diff --git a/node/Peer.hpp b/node/Peer.hpp index efe34825..8286bfff 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -121,7 +121,9 @@ public: bool hasActivePathTo(uint64_t now,const InetAddress &addr) const; /** - * If we have a confirmed path to this address, mark others as cluster suboptimal + * Set which known path for an address family is optimal + * + * This only modifies paths within the same address family * * @param addr Address to make exclusive */ @@ -161,7 +163,7 @@ public: * Send pings or keepalives depending on configured timeouts * * @param now Current time - * @param inetAddressFamily Keep this address family alive, or 0 to simply pick current best ignoring family + * @param inetAddressFamily Keep this address family alive, or -1 for any * @return True if we have at least one direct path */ bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); @@ -285,7 +287,7 @@ public: inline bool hasClusterOptimalPath(uint64_t now) const { for(unsigned int p=0,np=_numPaths;palive(now)) && (!_paths[p].clusterSuboptimal) ) + if ( (_paths[p].path->alive(now)) && ((_paths[p].clusterWeights & 1) != 0) ) return true; } return false; @@ -365,7 +367,9 @@ private: inline uint64_t _pathScore(const unsigned int p) const { - return ( (_paths[p].path->lastIn() + (_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK))) - ((ZT_PEER_PING_PERIOD * 10) * (uint64_t)_paths[p].clusterSuboptimal) ); + return ( _paths[p].path->lastIn() + + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + + (uint64_t)(_paths[p].clusterWeights * ZT_PEER_PING_PERIOD) ); } unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; @@ -384,9 +388,9 @@ private: uint16_t _vRevision; Identity _id; struct { - SharedPtr path; uint64_t lastReceive; - bool clusterSuboptimal; + SharedPtr path; + unsigned int clusterWeights; } _paths[ZT_MAX_PEER_NETWORK_PATHS]; Mutex _paths_m; unsigned int _numPaths; diff --git a/one.cpp b/one.cpp index 7cddb376..6ad5c8e6 100644 --- a/one.cpp +++ b/one.cpp @@ -330,12 +330,12 @@ static int cli(int argc,char **argv) out << "200 listpeers " << ZT_EOL_S; if (j.is_array()) { for(unsigned long k=0;k " << ZT_EOL_S; if (j.is_array()) { for(unsigned long i=0;i 0) aa.push_back(','); aa.append(addr); diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 5ed6b8b7..03c5942d 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -183,14 +183,14 @@ static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath * "%s\t\"address\": \"%s\",\n" "%s\t\"lastSend\": %llu,\n" "%s\t\"lastReceive\": %llu,\n" - "%s\t\"active\": %s,\n" + "%s\t\"active\": true,\n" "%s\t\"preferred\": %s,\n" "%s\t\"trustedPathId\": %llu\n" "%s}", prefix,_jsonEscape(reinterpret_cast(&(pp[i].address))->toString()).c_str(), prefix,pp[i].lastSend, prefix,pp[i].lastReceive, - prefix,(pp[i].active == 0) ? "false" : "true", + prefix, prefix,(pp[i].preferred == 0) ? "false" : "true", prefix,pp[i].trustedPathId, prefix); -- cgit v1.2.3 From d7f2287ce960b3a4917699e7c9c98ac26bfb8ca7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 5 Sep 2016 15:47:22 -0700 Subject: More tweaks to path behavior. --- node/Path.hpp | 10 ++-------- node/Peer.cpp | 39 +++++++++++++++++++++++++++------------ node/Peer.hpp | 4 ++-- node/Utils.cpp | 1 + service/ControlPlane.cpp | 4 ++-- 5 files changed, 34 insertions(+), 24 deletions(-) (limited to 'node') diff --git a/node/Path.hpp b/node/Path.hpp index d0e1d737..718e1cdd 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -152,16 +152,10 @@ public: inline InetAddress::IpScope ipScope() const { return _ipScope; } /** - * @return Preference rank, higher == better (will be less than 255) + * @return Preference rank, higher == better */ inline unsigned int preferenceRank() const { - /* First, since the scope enum values in InetAddress.hpp are in order of - * use preference rank, we take that. Then we multiple by two, yielding - * a sequence like 0, 2, 4, 6, etc. Then if it's IPv6 we add one. This - * makes IPv6 addresses of a given scope outrank IPv4 addresses of the - * same scope -- e.g. 1 outranks 0. This makes us prefer IPv6, but not - * if the address scope/class is of a fundamentally lower rank. */ return ( ((unsigned int)_ipScope << 1) | (unsigned int)(_addr.ss_family == AF_INET6) ); } @@ -213,7 +207,7 @@ public: /** * @return True if this path needs a heartbeat */ - inline bool needsHeartbeat(const uint64_t now) const { return ((now - _lastOut) > ZT_PATH_HEARTBEAT_PERIOD); } + inline bool needsHeartbeat(const uint64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); } /** * @return Last time we sent something diff --git a/node/Peer.cpp b/node/Peer.cpp index 0cb55fd8..3701d9a1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -134,24 +134,38 @@ void Peer::received( } } - if ((!pathIsConfirmed)&&(RR->node->shouldUsePathForZeroTierTraffic(path->localAddress(),path->address()))) { + if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(path->localAddress(),path->address())) ) { if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); - unsigned int slot = 0; + unsigned int slot; if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) { slot = _numPaths++; } else { + // First try to replace the worst within the same address family, if possible + int worstSlot = -1; uint64_t worstScore = 0xffffffffffffffffULL; - unsigned int worstPath = ZT_MAX_PEER_NETWORK_PATHS-1; for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _pathScore(p); - if (s < worstScore) { - worstScore = s; - worstPath = p; + if (_paths[p].path->address().ss_family == path->address().ss_family) { + const uint64_t s = _pathScore(p); + if (s < worstScore) { + worstScore = s; + worstSlot = (int)p; + } + } + } + if (worstSlot >= 0) { + slot = (unsigned int)worstSlot; + } else { + slot = ZT_MAX_PEER_NETWORK_PATHS - 1; + for(unsigned int p=0;p<_numPaths;++p) { + const uint64_t s = _pathScore(p); + if (s < worstScore) { + worstScore = s; + slot = p; + } } } - slot = worstPath; } _paths[slot].lastReceive = now; @@ -164,20 +178,21 @@ void Peer::received( _paths[slot].clusterWeights = 1; #endif } else { - - TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str()); + TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { + // Newer than 1.1.0 can use ECHO, which is smaller Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); outp.armor(_key,true); path->send(RR,outp.data(),outp.size(),now); } else { + // For backward compatibility we send HELLO to ancient nodes sendHELLO(path->localAddress(),path->address(),now); } - } } } else if (trustEstablished) { + // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) _pushDirectPaths(path,now); } @@ -286,7 +301,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if ((inetAddressFamily < 0)||(_paths[p].path->address().ss_family == inetAddressFamily)) { + if ((inetAddressFamily < 0)||((int)_paths[p].path->address().ss_family == inetAddressFamily)) { const uint64_t s = _pathScore(p); if (s >= best) { best = s; diff --git a/node/Peer.hpp b/node/Peer.hpp index 8286bfff..25ef72b7 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -164,7 +164,7 @@ public: * * @param now Current time * @param inetAddressFamily Keep this address family alive, or -1 for any - * @return True if we have at least one direct path + * @return True if we have at least one direct path of the given family (or any if family is -1) */ bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); @@ -367,7 +367,7 @@ private: inline uint64_t _pathScore(const unsigned int p) const { - return ( _paths[p].path->lastIn() + + return ( _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + (uint64_t)(_paths[p].clusterWeights * ZT_PEER_PING_PERIOD) ); } diff --git a/node/Utils.cpp b/node/Utils.cpp index 2d9515ee..9d67fc22 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -292,6 +292,7 @@ unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...) if ((n >= (int)len)||(n < 0)) { if (len) buf[len - 1] = (char)0; + abort(); throw std::length_error("buf[] overflow in Utils::snprintf"); } diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 03c5942d..a24e3eb4 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -165,7 +165,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_VirtualNetw static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath *pp,unsigned int count) { - char json[1024]; + char json[2048]; char prefix[32]; if (depth >= sizeof(prefix)) // sanity check -- shouldn't be possible @@ -201,7 +201,7 @@ static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath * static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) { - char json[1024]; + char json[2048]; char prefix[32]; if (depth >= sizeof(prefix)) // sanity check -- shouldn't be possible -- cgit v1.2.3 From 43780742b03ffa69fb1a1868f976ac03edce921b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 6 Sep 2016 11:10:04 -0700 Subject: comments, docs --- node/Path.hpp | 2 ++ node/Peer.cpp | 2 ++ 2 files changed, 4 insertions(+) (limited to 'node') diff --git a/node/Path.hpp b/node/Path.hpp index 718e1cdd..8151ed27 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -156,6 +156,8 @@ public: */ inline unsigned int preferenceRank() const { + // This causes us to rank paths in order of IP scope rank (see InetAdddress.hpp) but + // within each IP scope class to prefer IPv6 over IPv4. return ( ((unsigned int)_ipScope << 1) | (unsigned int)(_addr.ss_family == AF_INET6) ); } diff --git a/node/Peer.cpp b/node/Peer.cpp index 3701d9a1..110d67fd 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -138,6 +138,7 @@ void Peer::received( if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); + // Since this is a new path, figure out where to put it (possibly replacing an old/dead one) unsigned int slot; if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) { slot = _numPaths++; @@ -157,6 +158,7 @@ void Peer::received( if (worstSlot >= 0) { slot = (unsigned int)worstSlot; } else { + // If we can't find one with the same family, replace the worst of any family slot = ZT_MAX_PEER_NETWORK_PATHS - 1; for(unsigned int p=0;p<_numPaths;++p) { const uint64_t s = _pathScore(p); -- cgit v1.2.3 From 8a2e8bd5854619c9f701ad300724a1e58d4b6822 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 6 Sep 2016 12:45:28 -0700 Subject: Rework how paths are set as remote cluster preferred. The code is now clearer and cluster preference indications are now very sticky as they should be. --- node/Hashtable.hpp | 2 +- node/Peer.cpp | 38 +++++++++++--------------------------- node/Peer.hpp | 43 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 37 deletions(-) (limited to 'node') diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index c550191e..66f2990a 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -362,7 +362,7 @@ private: template static inline unsigned long _hc(const O &obj) { - return obj.hashCode(); + return (unsigned long)obj.hashCode(); } static inline unsigned long _hc(const uint64_t i) { diff --git a/node/Peer.cpp b/node/Peer.cpp index 110d67fd..ab287d05 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -33,7 +33,6 @@ namespace ZeroTier { static uint32_t _natKeepaliveBuf = 0; Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : - RR(renv), _lastUsed(0), _lastReceive(0), _lastUnicastFrame(0), @@ -41,6 +40,8 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastAnnouncedTo(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), + RR(renv), + _remoteClusterOptimal4(0), _vProto(0), _vMajor(0), _vMinor(0), @@ -50,6 +51,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _latency(0), _directPathPushCutoffCount(0) { + memset(_remoteClusterOptimal6,0,sizeof(_remoteClusterOptimal6)); if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw std::runtime_error("new peer identity key agreement failed"); } @@ -126,7 +128,7 @@ void Peer::received( _paths[p].lastReceive = now; _paths[p].path = path; // local address may have changed! #ifdef ZT_ENABLE_CLUSTER - _paths[p].clusterWeights = (unsigned int)(!suboptimalPath); + _paths[p].localClusterSuboptimal = suboptimalPath; #endif pathIsConfirmed = true; break; @@ -173,11 +175,9 @@ void Peer::received( _paths[slot].lastReceive = now; _paths[slot].path = path; #ifdef ZT_ENABLE_CLUSTER - _paths[slot].clusterWeights = (unsigned int)(!suboptimalPath); + _paths[p].localClusterSuboptimal = suboptimalPath; if (RR->cluster) RR->cluster->broadcastHavePeer(_id); -#else - _paths[slot].clusterWeights = 1; #endif } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); @@ -216,26 +216,6 @@ bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const return false; } -void Peer::setClusterOptimal(const InetAddress &addr) -{ - Mutex::Lock _l(_paths_m); - - int opt = -1; - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address() == addr) { - opt = (int)p; - break; - } - } - - if (opt >= 0) { // only change anything if we have the optimal path - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address().ss_family == addr.ss_family) - _paths[p].clusterWeights = ((int)p == opt) ? 2 : 0; - } - } -} - bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) { Mutex::Lock _l(_paths_m); @@ -350,7 +330,9 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) if (x != y) { _paths[y].lastReceive = _paths[x].lastReceive; _paths[y].path = _paths[x].path; - _paths[y].clusterWeights = _paths[x].clusterWeights; +#ifdef ZT_ENABLE_CLUSTER + _paths[y].localClusterSuboptimal = _paths[x].localClusterSuboptimal; +#endif } ++y; } @@ -399,7 +381,9 @@ void Peer::clean(uint64_t now) if (y != x) { _paths[y].lastReceive = _paths[x].lastReceive; _paths[y].path = _paths[x].path; - _paths[y].clusterWeights = _paths[x].clusterWeights; +#ifdef ZT_ENABLE_CLUSTER + _paths[y].localClusterSuboptimal = _paths[x].localClusterSuboptimal; +#endif } ++y; } diff --git a/node/Peer.hpp b/node/Peer.hpp index 25ef72b7..cd08e2d7 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -123,11 +123,16 @@ public: /** * Set which known path for an address family is optimal * - * This only modifies paths within the same address family - * * @param addr Address to make exclusive */ - void setClusterOptimal(const InetAddress &addr); + inline void setClusterOptimal(const InetAddress &addr) + { + if (addr.ss_family == AF_INET) { + _remoteClusterOptimal4 = (uint32_t)reinterpret_cast(&addr)->sin_addr.s_addr; + } else if (addr.ss_family == AF_INET6) { + memcpy(_remoteClusterOptimal6,reinterpret_cast(&addr)->sin6_addr.s6_addr,16); + } + } /** * Send via best direct path @@ -367,14 +372,30 @@ private: inline uint64_t _pathScore(const unsigned int p) const { - return ( _paths[p].lastReceive + - (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + - (uint64_t)(_paths[p].clusterWeights * ZT_PEER_PING_PERIOD) ); + uint64_t s = ZT_PEER_PING_PERIOD; + if (_paths[p].path->address().ss_family == AF_INET) { + s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + (uint64_t)(ZT_PEER_PING_PERIOD * (unsigned long)(reinterpret_cast(&(_paths[p].path->address()))->sin_addr.s_addr == _remoteClusterOptimal4)); + } else if (_paths[p].path->address().ss_family == AF_INET6) { + uint64_t clusterWeight = ZT_PEER_PING_PERIOD; + const uint8_t *a = reinterpret_cast(reinterpret_cast(&(_paths[p].path->address()))->sin6_addr.s6_addr); + for(long i=0;i<16;++i) { + if (a[i] != _remoteClusterOptimal6[i]) { + clusterWeight = 0; + break; + } + } + s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + clusterWeight; + } else { + s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); + } +#ifdef ZT_ENABLE_CLUSTER + s -= ZT_PEER_PING_PERIOD * (uint64_t)_paths[p].localClusterSuboptimal; +#endif + return s; } unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; - - const RuntimeEnvironment *RR; + uint8_t _remoteClusterOptimal6[16]; uint64_t _lastUsed; uint64_t _lastReceive; // direct or indirect uint64_t _lastUnicastFrame; @@ -382,6 +403,8 @@ private: uint64_t _lastAnnouncedTo; uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; + const RuntimeEnvironment *RR; + uint32_t _remoteClusterOptimal4; uint16_t _vProto; uint16_t _vMajor; uint16_t _vMinor; @@ -390,7 +413,9 @@ private: struct { uint64_t lastReceive; SharedPtr path; - unsigned int clusterWeights; +#ifdef ZT_ENABLE_CLUSTER + bool localClusterSuboptimal; +#endif } _paths[ZT_MAX_PEER_NETWORK_PATHS]; Mutex _paths_m; unsigned int _numPaths; -- cgit v1.2.3 From 48a374c82c89b69a71d1922c4396265394e9045f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 6 Sep 2016 14:05:58 -0700 Subject: (1) fix crazy bug introduced in doRENDEZVOUS(), (2) reclaim Paths after paths[] condense, (3) fix an edge case around symmetric NAT and external IP change detection. --- node/IncomingPacket.cpp | 12 ++++++------ node/Path.hpp | 3 ++- node/Peer.cpp | 6 +++++- 3 files changed, 13 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index a84b2beb..3d2d586e 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -283,7 +283,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer // VALID -- if we made it here, packet passed identity and authenticity checks! } - if (externalSurfaceAddress) + if ((externalSurfaceAddress)&&(hops() == 0)) RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now()); Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); @@ -391,7 +391,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p peer->addDirectLatencyMeasurment(latency); peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); - if (externalSurfaceAddress) + if ((externalSurfaceAddress)&&(hops() == 0)) RR->sa->iam(peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; @@ -516,8 +516,8 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< { try { const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr withPeer(RR->topology->getPeer(with)); - if (withPeer) { + const SharedPtr rendezvousWith(RR->topology->getPeer(with)); + if (rendezvousWith) { const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { @@ -525,8 +525,8 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< if (!RR->topology->isUpstream(peer->identity())) { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { - RR->node->putPacket(_path->localAddress(),atAddr,"NATSUX",6,2); // send low-TTL packet to 'open' local NAT(s) - peer->sendHELLO(_path->localAddress(),atAddr,RR->node->now()); + RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->sendHELLO(_path->localAddress(),atAddr,RR->node->now()); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); diff --git a/node/Path.hpp b/node/Path.hpp index 8151ed27..129913e1 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -29,6 +29,7 @@ #include "InetAddress.hpp" #include "SharedPtr.hpp" #include "AtomicCounter.hpp" +#include "NonCopyable.hpp" /** * Maximum return value of preferenceRank() @@ -42,7 +43,7 @@ class RuntimeEnvironment; /** * A path across the physical network */ -class Path +class Path : NonCopyable { friend class SharedPtr; diff --git a/node/Peer.cpp b/node/Peer.cpp index ab287d05..c56dbca9 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -339,7 +339,9 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) ++x; } _numPaths = y; - return (y < np); + while (y < ZT_MAX_PEER_NETWORK_PATHS) + _paths[y++].path.zero(); // let go of unused SmartPtr<>'s + return (_numPaths < np); } void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const @@ -390,6 +392,8 @@ void Peer::clean(uint64_t now) ++x; } _numPaths = y; + while (y < ZT_MAX_PEER_NETWORK_PATHS) + _paths[y++].path.zero(); // let go of unused SmartPtr<>'s } bool Peer::_pushDirectPaths(const SharedPtr &path,uint64_t now) -- cgit v1.2.3 From f2d2df2b112c8a644b718abc521af296a83b5337 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 6 Sep 2016 15:06:07 -0700 Subject: Cluster build fix. --- node/Cluster.cpp | 2 +- node/Peer.cpp | 2 +- node/Peer.hpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 933bfb37..2a261e51 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -361,7 +361,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) case CLUSTER_MESSAGE_WANT_PEER: { const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; SharedPtr peer(RR->topology->getPeerNoCache(zeroTierAddress)); - if ( (peer) && (peer->hasClusterOptimalPath(RR->node->now())) ) { + if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) { Buffer<1024> buf; peer->identity().serialize(buf); Mutex::Lock _l2(_members[fromMemberId].lock); diff --git a/node/Peer.cpp b/node/Peer.cpp index c56dbca9..abbbea72 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -175,7 +175,7 @@ void Peer::received( _paths[slot].lastReceive = now; _paths[slot].path = path; #ifdef ZT_ENABLE_CLUSTER - _paths[p].localClusterSuboptimal = suboptimalPath; + _paths[slot].localClusterSuboptimal = suboptimalPath; if (RR->cluster) RR->cluster->broadcastHavePeer(_id); #endif diff --git a/node/Peer.hpp b/node/Peer.hpp index cd08e2d7..3ffabb05 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -289,10 +289,10 @@ public: * @param now Current time * @return True if this peer has at least one active direct path that is not cluster-suboptimal */ - inline bool hasClusterOptimalPath(uint64_t now) const + inline bool hasLocalClusterOptimalPath(uint64_t now) const { for(unsigned int p=0,np=_numPaths;palive(now)) && ((_paths[p].clusterWeights & 1) != 0) ) + if ( (_paths[p].path->alive(now)) && (!_paths[p].localClusterSuboptimal) ) return true; } return false; -- cgit v1.2.3 From b5c86b6ba4112b23e46170fe241b4688532b493e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 11:13:17 -0700 Subject: Bunch more path refactoring. Peers no longer forget paths, but do not normally use expired paths. Expired paths might still be tried if nothing else is reachable. --- include/ZeroTierOne.h | 5 ++ node/Constants.hpp | 18 ++++++-- node/IncomingPacket.cpp | 2 +- node/Node.cpp | 25 ++++------ node/Peer.cpp | 117 ++++++++++++++++++----------------------------- node/Peer.hpp | 46 ++++++++++--------- node/SelfAwareness.cpp | 22 ++++----- node/SelfAwareness.hpp | 1 - node/Switch.cpp | 22 ++++++--- node/Topology.cpp | 5 +- service/ControlPlane.cpp | 6 ++- 11 files changed, 126 insertions(+), 143 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 583abfe4..0c22ae9d 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1053,6 +1053,11 @@ typedef struct */ uint64_t trustedPathId; + /** + * Is path expired? + */ + int expired; + /** * Is path preferred? */ diff --git a/node/Constants.hpp b/node/Constants.hpp index ea4c434d..67e6fb58 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -244,17 +244,22 @@ * This is also how often pings will be retried to upstream peers (relays, roots) * constantly until something is heard. */ -#define ZT_PING_CHECK_INVERVAL 9000 +#define ZT_PING_CHECK_INVERVAL 10000 /** * How frequently to send heartbeats over in-use paths */ -#define ZT_PATH_HEARTBEAT_PERIOD 15000 +#define ZT_PATH_HEARTBEAT_PERIOD 10000 /** * Paths are considered inactive if they have not received traffic in this long */ -#define ZT_PATH_ALIVE_TIMEOUT 35000 +#define ZT_PATH_ALIVE_TIMEOUT 25000 + +/** + * Minimum time between attempts to check dead paths to see if they can be re-awakened + */ +#define ZT_PATH_MIN_REACTIVATE_INTERVAL 2500 /** * Delay between full-fledge pings of directly connected peers @@ -262,10 +267,15 @@ #define ZT_PEER_PING_PERIOD 60000 /** - * Peers forget paths that have not spoken in this long + * Paths are considered expired if they have not produced a real packet in this long */ #define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) +/** + * How often to retry expired paths that we're still remembering + */ +#define ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD (ZT_PEER_PING_PERIOD * 10) + /** * Timeout for overall peer activity (measured from last receive) */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 3d2d586e..891607ed 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1163,7 +1163,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt remainingHopsPtr += ZT_ADDRESS_LENGTH; SharedPtr nhp(RR->topology->getPeer(nextHop[h])); if (nhp) { - SharedPtr nhbp(nhp->getBestPath(now)); + SharedPtr nhbp(nhp->getBestPath(now,false)); if ((nhbp)&&(nhbp->alive(now))) nextHopBestPathAddress[h] = nhbp->address(); } diff --git a/node/Node.cpp b/node/Node.cpp index a7d4cfa9..edd48575 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -202,14 +202,6 @@ public: } } - if (!upstream) { - // If I am a root server, only ping other root servers -- roots don't ping "down" - // since that would just be a waste of bandwidth and could potentially cause route - // flapping in Cluster mode. - if (RR->topology->amRoot()) - return; - } - if (upstream) { // "Upstream" devices are roots and relays and get special treatment -- they stay alive // forever and we try to keep (if available) both IPv4 and IPv6 channels open to them. @@ -415,15 +407,16 @@ ZT_PeerList *Node::peers() const p->latency = pi->second->latency(); p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF; - std::vector< SharedPtr > paths(pi->second->paths()); - SharedPtr bestp(pi->second->getBestPath(_now)); + std::vector< std::pair< SharedPtr,bool > > paths(pi->second->paths(_now)); + SharedPtr bestp(pi->second->getBestPath(_now,false)); p->pathCount = 0; - for(std::vector< SharedPtr >::iterator path(paths.begin());path!=paths.end();++path) { - memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); - p->paths[p->pathCount].lastSend = (*path)->lastOut(); - p->paths[p->pathCount].lastReceive = (*path)->lastIn(); - p->paths[p->pathCount].preferred = (*path == bestp) ? 1 : 0; - p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); + for(std::vector< std::pair< SharedPtr,bool > >::iterator path(paths.begin());path!=paths.end();++path) { + memcpy(&(p->paths[p->pathCount].address),&(path->first->address()),sizeof(struct sockaddr_storage)); + p->paths[p->pathCount].lastSend = path->first->lastOut(); + p->paths[p->pathCount].lastReceive = path->first->lastIn(); + p->paths[p->pathCount].expired = path->second; + p->paths[p->pathCount].preferred = (path->first == bestp) ? 1 : 0; + p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->first->address()); ++p->pathCount; } } diff --git a/node/Peer.cpp b/node/Peer.cpp index abbbea72..a2a91769 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -27,6 +27,14 @@ #include "Cluster.hpp" #include "Packet.hpp" +#ifndef AF_MAX +#if AF_INET > AF_INET6 +#define AF_MAX AF_INET +#else +#define AF_MAX AF_INET6 +#endif +#endif + namespace ZeroTier { // Used to send varying values for NAT keepalive @@ -150,7 +158,7 @@ void Peer::received( uint64_t worstScore = 0xffffffffffffffffULL; for(unsigned int p=0;p<_numPaths;++p) { if (_paths[p].path->address().ss_family == path->address().ss_family) { - const uint64_t s = _pathScore(p); + const uint64_t s = _pathScore(p,now); if (s < worstScore) { worstScore = s; worstSlot = (int)p; @@ -163,7 +171,7 @@ void Peer::received( // If we can't find one with the same family, replace the worst of any family slot = ZT_MAX_PEER_NETWORK_PATHS - 1; for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _pathScore(p); + const uint64_t s = _pathScore(p,now); if (s < worstScore) { worstScore = s; slot = p; @@ -210,7 +218,7 @@ bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const { Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { - if ( (_paths[p].path->address() == addr) && (_paths[p].path->alive(now)) ) + if ( (_paths[p].path->address() == addr) && ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)) ) return true; } return false; @@ -223,8 +231,8 @@ bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceE int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->alive(now)||(forceEvenIfDead)) { - const uint64_t s = _pathScore(p); + if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)||(forceEvenIfDead)) ) { + const uint64_t s = _pathScore(p,now); if (s >= best) { best = s; bestp = (int)p; @@ -239,17 +247,19 @@ bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceE } } -SharedPtr Peer::getBestPath(uint64_t now) +SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) { Mutex::Lock _l(_paths_m); int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _pathScore(p); - if (s >= best) { - best = s; - bestp = (int)p; + if ( ((now - _paths[p].lastReceive) < ZT_PEER_PATH_EXPIRATION) || (includeExpired) ) { + const uint64_t s = _pathScore(p,now); + if (s >= best) { + best = s; + bestp = (int)p; + } } } @@ -283,8 +293,8 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if ((inetAddressFamily < 0)||((int)_paths[p].path->address().ss_family == inetAddressFamily)) { - const uint64_t s = _pathScore(p); + if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && ((inetAddressFamily < 0)||((int)_paths[p].path->address().ss_family == inetAddressFamily)) ) { + const uint64_t s = _pathScore(p,now); if (s >= best) { best = s; bestp = (int)p; @@ -293,7 +303,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) } if (bestp >= 0) { - if ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) { + if ((now - _paths[best].lastReceive) >= ZT_PEER_PING_PERIOD) { sendHELLO(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); } else if (_paths[bestp].path->needsHeartbeat(now)) { _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads @@ -309,39 +319,24 @@ bool Peer::hasActiveDirectPath(uint64_t now) const { Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->alive(now)) + if (((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION)&&(_paths[p].path->alive(now))) return true; } return false; } -bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now) +bool Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) { Mutex::Lock _l(_paths_m); - unsigned int np = _numPaths; - unsigned int x = 0; - unsigned int y = 0; - while (x < np) { - if (_paths[x].path->address().ipScope() == scope) { - // Resetting a path means sending a HELLO and then forgetting it. If we - // get OK(HELLO) then it will be re-learned. - sendHELLO(_paths[x].path->localAddress(),_paths[x].path->address(),now); - } else { - if (x != y) { - _paths[y].lastReceive = _paths[x].lastReceive; - _paths[y].path = _paths[x].path; -#ifdef ZT_ENABLE_CLUSTER - _paths[y].localClusterSuboptimal = _paths[x].localClusterSuboptimal; -#endif - } - ++y; + bool resetSomething = false; + for(unsigned int p=0;p<_numPaths;++p) { + if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { + sendHELLO(_paths[p].path->localAddress(),_paths[p].path->address(),now); + _paths[p].lastReceive >>= 2; // de-prioritize heavily vs. other paths, will get reset if we get OK(HELLO) or other traffic + resetSomething = true; } - ++x; } - _numPaths = y; - while (y < ZT_MAX_PEER_NETWORK_PATHS) - _paths[y++].path.zero(); // let go of unused SmartPtr<>'s - return (_numPaths < np); + return resetSomething; } void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const @@ -351,17 +346,19 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) int bestp4 = -1,bestp6 = -1; uint64_t best4 = 0ULL,best6 = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address().ss_family == AF_INET) { - const uint64_t s = _pathScore(p); - if (s >= best4) { - best4 = s; - bestp4 = (int)p; - } - } else if (_paths[p].path->address().ss_family == AF_INET6) { - const uint64_t s = _pathScore(p); - if (s >= best6) { - best6 = s; - bestp6 = (int)p; + if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)) ) { + if (_paths[p].path->address().ss_family == AF_INET) { + const uint64_t s = _pathScore(p,now); + if (s >= best4) { + best4 = s; + bestp4 = (int)p; + } + } else if (_paths[p].path->address().ss_family == AF_INET6) { + const uint64_t s = _pathScore(p,now); + if (s >= best6) { + best6 = s; + bestp6 = (int)p; + } } } } @@ -372,30 +369,6 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) v6 = _paths[bestp6].path->address(); } -void Peer::clean(uint64_t now) -{ - Mutex::Lock _l(_paths_m); - unsigned int np = _numPaths; - unsigned int x = 0; - unsigned int y = 0; - while (x < np) { - if ((now - _paths[x].lastReceive) <= ZT_PEER_PATH_EXPIRATION) { - if (y != x) { - _paths[y].lastReceive = _paths[x].lastReceive; - _paths[y].path = _paths[x].path; -#ifdef ZT_ENABLE_CLUSTER - _paths[y].localClusterSuboptimal = _paths[x].localClusterSuboptimal; -#endif - } - ++y; - } - ++x; - } - _numPaths = y; - while (y < ZT_MAX_PEER_NETWORK_PATHS) - _paths[y++].path.zero(); // let go of unused SmartPtr<>'s -} - bool Peer::_pushDirectPaths(const SharedPtr &path,uint64_t now) { #ifdef ZT_ENABLE_CLUSTER diff --git a/node/Peer.hpp b/node/Peer.hpp index 3ffabb05..6e1d378f 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -149,9 +149,10 @@ public: * Get the best current direct path * * @param now Current time + * @param includeDead If true, include even expired paths * @return Best current path or NULL if none */ - SharedPtr getBestPath(uint64_t now); + SharedPtr getBestPath(uint64_t now,bool includeExpired); /** * Send a HELLO to this peer at a specified physical address @@ -175,18 +176,22 @@ public: /** * @param now Current time - * @return True if this peer has at least one active direct path + * @return True if this peer has at least one active and alive direct path */ bool hasActiveDirectPath(uint64_t now) const; /** - * Reset paths within a given scope + * Reset paths within a given IP scope and address family * - * @param scope IP scope of paths to reset + * Resetting a path involves sending a HELLO to it and then de-prioritizing + * it vs. other paths. + * + * @param scope IP scope + * @param inetAddressFamily Family e.g. AF_INET * @param now Current time - * @return True if at least one path was forgotten + * @return True if we forgot at least one path */ - bool resetWithinScope(InetAddress::IpScope scope,uint64_t now); + bool resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); /** * Get most recently active path addresses for IPv4 and/or IPv6 @@ -201,21 +206,15 @@ public: void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; /** - * Perform periodic cleaning operations - * * @param now Current time + * @return All known direct paths to this peer and whether they are expired (true == expired) */ - void clean(uint64_t now); - - /** - * @return All known direct paths to this peer (active or inactive) - */ - inline std::vector< SharedPtr > paths() const + inline std::vector< std::pair< SharedPtr,bool > > paths(const uint64_t now) const { - std::vector< SharedPtr > pp; + std::vector< std::pair< SharedPtr,bool > > pp; Mutex::Lock _l(_paths_m); for(unsigned int p=0,np=_numPaths;p,bool >(_paths[p].path,(now - _paths[p].lastReceive) > ZT_PEER_PATH_EXPIRATION)); return pp; } @@ -370,11 +369,12 @@ public: private: bool _pushDirectPaths(const SharedPtr &path,uint64_t now); - inline uint64_t _pathScore(const unsigned int p) const + inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const { - uint64_t s = ZT_PEER_PING_PERIOD; + uint64_t s = ZT_PEER_PING_PERIOD + _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); + if (_paths[p].path->address().ss_family == AF_INET) { - s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + (uint64_t)(ZT_PEER_PING_PERIOD * (unsigned long)(reinterpret_cast(&(_paths[p].path->address()))->sin_addr.s_addr == _remoteClusterOptimal4)); + s += (uint64_t)(ZT_PEER_PING_PERIOD * (unsigned long)(reinterpret_cast(&(_paths[p].path->address()))->sin_addr.s_addr == _remoteClusterOptimal4)); } else if (_paths[p].path->address().ss_family == AF_INET6) { uint64_t clusterWeight = ZT_PEER_PING_PERIOD; const uint8_t *a = reinterpret_cast(reinterpret_cast(&(_paths[p].path->address()))->sin6_addr.s6_addr); @@ -384,13 +384,15 @@ private: break; } } - s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)) + clusterWeight; - } else { - s += _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); + s += clusterWeight; } + + s += (ZT_PEER_PING_PERIOD / 2) * (uint64_t)_paths[p].path->alive(now); + #ifdef ZT_ENABLE_CLUSTER s -= ZT_PEER_PING_PERIOD * (uint64_t)_paths[p].localClusterSuboptimal; #endif + return s; } diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index b9ab9d67..6bf50720 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -33,37 +33,31 @@ #include "Switch.hpp" // Entry timeout -- make it fairly long since this is just to prevent stale buildup -#define ZT_SELFAWARENESS_ENTRY_TIMEOUT 3600000 +#define ZT_SELFAWARENESS_ENTRY_TIMEOUT 600000 namespace ZeroTier { class _ResetWithinScope { public: - _ResetWithinScope(uint64_t now,InetAddress::IpScope scope) : + _ResetWithinScope(uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : _now(now), + _family(inetAddressFamily), _scope(scope) {} - inline void operator()(Topology &t,const SharedPtr &p) - { - if (p->resetWithinScope(_scope,_now)) - peersReset.push_back(p); - } + inline void operator()(Topology &t,const SharedPtr &p) { if (p->resetWithinScope(_scope,_family,_now)) peersReset.push_back(p); } std::vector< SharedPtr > peersReset; private: uint64_t _now; + int _family; InetAddress::IpScope _scope; }; SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) : RR(renv), - _phy(32) -{ -} - -SelfAwareness::~SelfAwareness() + _phy(128) { } @@ -98,8 +92,8 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc } } - // Reset all paths within this scope - _ResetWithinScope rset(now,(InetAddress::IpScope)scope); + // Reset all paths within this scope and address family + _ResetWithinScope rset(now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope); RR->topology->eachPeer<_ResetWithinScope &>(rset); // Send a NOP to all peers for whom we forgot a path. This will cause direct diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index c7bde87e..4bdafeb2 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -36,7 +36,6 @@ class SelfAwareness { public: SelfAwareness(const RuntimeEnvironment *renv); - ~SelfAwareness(); /** * Called when a trusted remote peer informs us of our external network address diff --git a/node/Switch.cpp b/node/Switch.cpp index 28a2564b..21d0b3c9 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -747,14 +747,20 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread bool Switch::_trySend(const Packet &packet,bool encrypt) { - SharedPtr peer(RR->topology->getPeer(packet.destination())); - + const SharedPtr peer(RR->topology->getPeer(packet.destination())); if (peer) { const uint64_t now = RR->node->now(); - SharedPtr viaPath(peer->getBestPath(now)); + // First get the best path, and if it's dead (and this is not a root) + // we attempt to re-activate that path but this packet will flow + // upstream. If the path comes back alive, it will be used in the future. + // For roots we don't do the alive check since roots are not required + // to send heartbeats "down" and because we have to at least try to + // go somewhere. + + SharedPtr viaPath(peer->getBestPath(now,false)); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) { - if ((now - viaPath->lastOut()) > 5000) { + if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) >> 2,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ECHO); outp.armor(peer->key(),true); viaPath->send(RR,outp.data(),outp.size(),now); @@ -763,8 +769,10 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) } if (!viaPath) { SharedPtr relay(RR->topology->getBestRoot()); - if ( (!relay) || (!(viaPath = relay->getBestPath(now))) ) - return false; + if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { + if (!(viaPath = peer->getBestPath(now,true))) + return false; + } } Packet tmp(packet); @@ -787,7 +795,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) unsigned int fragsRemaining = (remaining / (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)); if ((fragsRemaining * (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining) ++fragsRemaining; - unsigned int totalFragments = fragsRemaining + 1; + const unsigned int totalFragments = fragsRemaining + 1; for(unsigned int fno=1;fno *p = (SharedPtr *)0; while (i.next(a,p)) { - if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) { + if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) _peers.erase(*a); - } else { - (*p)->clean(now); - } } } { diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index a24e3eb4..b443a7fa 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -183,14 +183,16 @@ static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath * "%s\t\"address\": \"%s\",\n" "%s\t\"lastSend\": %llu,\n" "%s\t\"lastReceive\": %llu,\n" - "%s\t\"active\": true,\n" + "%s\t\"active\": %s,\n" + "%s\t\"expired\": %s,\n" "%s\t\"preferred\": %s,\n" "%s\t\"trustedPathId\": %llu\n" "%s}", prefix,_jsonEscape(reinterpret_cast(&(pp[i].address))->toString()).c_str(), prefix,pp[i].lastSend, prefix,pp[i].lastReceive, - prefix, + prefix,(pp[i].expired != 0) ? "false" : "true", + prefix,(pp[i].expired == 0) ? "false" : "true", prefix,(pp[i].preferred == 0) ? "false" : "true", prefix,pp[i].trustedPathId, prefix); -- cgit v1.2.3 From ff9f8b1c2b12cf11205a040748cbc9f223a67e19 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 11:15:36 -0700 Subject: Typo fix. --- node/Peer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index a2a91769..c24d4246 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -303,7 +303,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) } if (bestp >= 0) { - if ((now - _paths[best].lastReceive) >= ZT_PEER_PING_PERIOD) { + if ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) { sendHELLO(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); } else if (_paths[bestp].path->needsHeartbeat(now)) { _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads -- cgit v1.2.3 From a7d988745bcca4a0f9c838ec493e658b098d241d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 12:01:03 -0700 Subject: Use ECHO instead of HELLO where possible. --- node/IncomingPacket.cpp | 6 +++--- node/Path.hpp | 7 +++++++ node/Peer.cpp | 31 ++++++++++++++++++------------- node/Peer.hpp | 11 +++++++++++ 4 files changed, 39 insertions(+), 16 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 891607ed..0857fb3d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -526,7 +526,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->sendHELLO(_path->localAddress(),atAddr,RR->node->now()); + rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -1050,7 +1050,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->sendHELLO(InetAddress(),a,now); + peer->attemptToContactAt(InetAddress(),a,now); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1069,7 +1069,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->sendHELLO(InetAddress(),a,now); + peer->attemptToContactAt(InetAddress(),a,now); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } diff --git a/node/Path.hpp b/node/Path.hpp index 129913e1..27cff645 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -137,6 +137,13 @@ public: */ bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now); + /** + * Manually update last sent time + * + * @param t Time of send + */ + inline void sent(const uint64_t t) { _lastOut = t; } + /** * @return Address of local side of this path or NULL if unspecified */ diff --git a/node/Peer.cpp b/node/Peer.cpp index c24d4246..3d3ca247 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -189,16 +189,8 @@ void Peer::received( #endif } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - - if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { - // Newer than 1.1.0 can use ECHO, which is smaller - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); - outp.armor(_key,true); - path->send(RR,outp.data(),outp.size(),now); - } else { - // For backward compatibility we send HELLO to ancient nodes - sendHELLO(path->localAddress(),path->address(),now); - } + attemptToContactAt(path->localAddress(),path->address(),now); + path->sent(now); } } } else if (trustEstablished) { @@ -254,7 +246,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) int bestp = -1; uint64_t best = 0ULL; for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) < ZT_PEER_PATH_EXPIRATION) || (includeExpired) ) { + if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) || (includeExpired) ) { const uint64_t s = _pathScore(p,now); if (s >= best) { best = s; @@ -286,6 +278,17 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } +void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now) +{ + if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); + outp.armor(_key,true); + RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + } else { + sendHELLO(localAddr,atAddress,now); + } +} + bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); @@ -304,7 +307,8 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) if (bestp >= 0) { if ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) { - sendHELLO(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); + attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); + _paths[bestp].path->sent(now); } else if (_paths[bestp].path->needsHeartbeat(now)) { _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads _paths[bestp].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now); @@ -331,7 +335,8 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uin bool resetSomething = false; for(unsigned int p=0;p<_numPaths;++p) { if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - sendHELLO(_paths[p].path->localAddress(),_paths[p].path->address(),now); + attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now); + _paths[p].path->sent(now); _paths[p].lastReceive >>= 2; // de-prioritize heavily vs. other paths, will get reset if we get OK(HELLO) or other traffic resetSomething = true; } diff --git a/node/Peer.hpp b/node/Peer.hpp index 6e1d378f..7a7453ae 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -165,6 +165,17 @@ public: */ void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now); + /** + * Send ECHO (or HELLO for older peers) to this peer at the given address + * + * No statistics or sent times are updated here. + * + * @param localAddr Local address + * @param atAddress Destination address + * @param now Current time + */ + void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now); + /** * Send pings or keepalives depending on configured timeouts * -- cgit v1.2.3 From c9ee8612e496d833b287f00c548f76ee5879bfef Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 12:12:52 -0700 Subject: Credential TTL (tags/capabilities) should be credential time max delta, since we could get pushed one that is newer. --- controller/EmbeddedNetworkController.cpp | 12 ++++++------ node/Membership.hpp | 5 +++-- node/NetworkConfig.cpp | 4 ++-- node/NetworkConfig.hpp | 16 ++++++++-------- 4 files changed, 19 insertions(+), 18 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ff2f34ec..cf6bd7c9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -654,16 +654,16 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // for both.) This is computed by reference to the last time we deauthorized // a member, since within the time period since this event any temporal // differences are not particularly relevant. - uint64_t credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL; + uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; if (now > nmi.mostRecentDeauthTime) - credentialTtl += (now - nmi.mostRecentDeauthTime); - if (credentialTtl > ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL) - credentialTtl = ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL; + credentialtmd += (now - nmi.mostRecentDeauthTime); + if (credentialtmd > ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) + credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; - nc.credentialTimeToLive = credentialTtl; + nc.credentialTimeMaxDelta = credentialtmd; nc.revision = _jI(network["revision"],0ULL); nc.issuedTo = identity.address(); if (_jB(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; @@ -925,7 +925,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } if (_jB(network["private"],true)) { - CertificateOfMembership com(now,credentialTtl,nwid,identity.address()); + CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); if (com.sign(signingId)) { nc.com = com; } else { diff --git a/node/Membership.hpp b/node/Membership.hpp index 5e5efc50..209f6158 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -144,7 +144,7 @@ public: } /** - * Check whether a capability or tag is expired + * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time * * @param cred Credential to check -- must have timestamp() accessor method * @return True if credential is NOT expired @@ -153,7 +153,8 @@ public: inline bool isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred) const { const uint64_t ts = cred.timestamp(); - return ( ( (ts >= nconf.timestamp) || ((nconf.timestamp - ts) <= nconf.credentialTimeToLive) ) && (ts > _blacklistBefore) ); + const uint64_t delta = (ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts); + return ((delta <= nconf.credentialTimeMaxDelta)&&(ts > _blacklistBefore)); } /** diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 0c9c05ca..6acc48ea 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -37,7 +37,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,this->credentialTimeToLive)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; @@ -193,7 +193,7 @@ bool NetworkConfig::fromDictionary(const Dictionarytimestamp = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,0); - this->credentialTimeToLive = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL,0); + this->credentialTimeMaxDelta = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,0); this->revision = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REVISION,0); this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0); if (!this->issuedTo) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index e2bacb07..b5ab9ccb 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -41,12 +41,12 @@ #include "Identity.hpp" /** - * Default maximum credential TTL and maxDelta for COM timestamps + * Default maximum time delta for COMs, tags, and capabilities * * The current value is two hours, providing ample time for a controller to * experience fail-over, etc. */ -#define ZT_NETWORKCONFIG_DEFAULT_MAX_CREDENTIAL_TTL 7200000ULL +#define ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA 7200000ULL /** * Default minimum credential TTL and maxDelta for COM timestamps @@ -54,7 +54,7 @@ * This is just slightly over three minutes and provides three retries for * all currently online members to refresh. */ -#define ZT_NETWORKCONFIG_DEFAULT_MIN_CREDENTIAL_TTL 185000ULL +#define ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA 185000ULL /** * Flag: allow passive bridging (experimental) @@ -148,8 +148,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_TYPE "t" // text #define ZT_NETWORKCONFIG_DICT_KEY_NAME "n" -// credential time to live in ms -#define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TTL "cttl" +// credential time max delta in ms +#define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA "ctmd" // binary serialized certificate of membership #define ZT_NETWORKCONFIG_DICT_KEY_COM "C" // specialists (binary array of uint64_t) @@ -372,7 +372,7 @@ public: { printf("networkId==%.16llx\n",networkId); printf("timestamp==%llu\n",timestamp); - printf("credentialTimeToLive==%llu\n",credentialTimeToLive); + printf("credentialTimeMaxDelta==%llu\n",credentialTimeMaxDelta); printf("revision==%llu\n",revision); printf("issuedTo==%.10llx\n",issuedTo.toInt()); printf("multicastLimit==%u\n",multicastLimit); @@ -407,9 +407,9 @@ public: uint64_t timestamp; /** - * TTL for capabilities and tags + * Max difference between timestamp and tag/capability timestamp */ - uint64_t credentialTimeToLive; + uint64_t credentialTimeMaxDelta; /** * Controller-side revision counter for this configuration -- cgit v1.2.3 From 1c08f5e8578fa05975e9885212b6bd4583b397dc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 12:25:19 -0700 Subject: Tweak some expire times. --- node/Membership.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Membership.hpp b/node/Membership.hpp index 209f6158..293fa4d8 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -32,10 +32,10 @@ #include "NetworkConfig.hpp" // Expiration time for capability and tag cache -#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME 600000 +#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA -// Expiration time for Memberships (used in Peer::clean()) -#define ZT_MEMBERSHIP_EXPIRATION_TIME (ZT_MEMBERSHIP_STATE_EXPIRATION_TIME * 2) +// Expiration time for Memberships (used in Network::clean()) +#define ZT_MEMBERSHIP_EXPIRATION_TIME ZT_PEER_IN_MEMORY_EXPIRATION namespace ZeroTier { -- cgit v1.2.3 From 1908aa55f51d63bceb7ed5d4211a4274d732de63 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 15:15:52 -0700 Subject: Refactor MULTICAST_LIKE pushing to eliminate redundant and unnecessary pushes and simplify code. --- node/Constants.hpp | 28 +++++------- node/IncomingPacket.cpp | 93 +++++++++++++++++++++------------------ node/Membership.cpp | 7 ++- node/Membership.hpp | 49 ++++++++++++--------- node/Network.cpp | 115 ++++++++++++++++++++++-------------------------- node/Network.hpp | 46 ++++++------------- node/Node.cpp | 6 +-- node/Peer.cpp | 21 +++------ node/Peer.hpp | 19 ++++---- 9 files changed, 182 insertions(+), 202 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 67e6fb58..a625b480 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -180,14 +180,14 @@ #define ZT_PEER_SECRET_KEY_LENGTH 32 /** - * How often Topology::clean() and Network::clean() and similar are called, in ms + * Minimum delay between timer task checks to prevent thrashing */ -#define ZT_HOUSEKEEPING_PERIOD 120000 +#define ZT_CORE_TIMER_TASK_GRANULARITY 500 /** - * Overriding granularity for timer tasks to prevent CPU-intensive thrashing on every packet + * How often Topology::clean() and Network::clean() and similar are called, in ms */ -#define ZT_CORE_TIMER_TASK_GRANULARITY 500 +#define ZT_HOUSEKEEPING_PERIOD 120000 /** * How long to remember peer records in RAM if they haven't been used @@ -226,6 +226,11 @@ */ #define ZT_MULTICAST_LIKE_EXPIRE 600000 +/** + * Period for multicast LIKE announcements + */ +#define ZT_MULTICAST_ANNOUNCE_PERIOD 120000 + /** * Delay between explicit MULTICAST_GATHER requests for a given multicast channel */ @@ -239,12 +244,9 @@ #define ZT_MULTICAST_TRANSMIT_TIMEOUT 5000 /** - * Delay between scans of the topology active peer DB for peers that need ping - * - * This is also how often pings will be retried to upstream peers (relays, roots) - * constantly until something is heard. + * Delay between checks of peer pings, etc., and also related housekeeping tasks */ -#define ZT_PING_CHECK_INVERVAL 10000 +#define ZT_PING_CHECK_INVERVAL 5000 /** * How frequently to send heartbeats over in-use paths @@ -298,14 +300,6 @@ */ #define ZT_MIN_UNITE_INTERVAL 30000 -/** - * Delay between initial direct NAT-t packet and more aggressive techniques - * - * This may also be a delay before sending the first packet if we determine - * that we should wait for the remote to initiate rendezvous first. - */ -#define ZT_NAT_T_TACTICAL_ESCALATION_DELAY 1000 - /** * Sanity limit on maximum bridge routes * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 0857fb3d..97e1abe3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -42,6 +42,8 @@ namespace ZeroTier { +static const SharedPtr NULL_NETWORK; + bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) { const Address sourceAddress(source()); @@ -88,7 +90,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false,NULL_NETWORK); return true; case Packet::VERB_HELLO: return _doHELLO(RR,peer); @@ -172,7 +174,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); + peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -339,7 +341,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer _path->send(RR,outp.data(),outp.size(),RR->node->now()); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -461,7 +463,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); + peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -505,7 +507,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -537,7 +539,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< } else { TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } - peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -547,25 +549,27 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID))); + const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(nwid)); + bool approved = false; if (network) { if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { if (!network->isAllowed(peer)) { TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,false); } else { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); - const MAC sourceMac(peer->address(),network->id()); + const MAC sourceMac(peer->address(),nwid); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) - RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true); + RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); + approved = true; // this means approved on the network in general, not this packet per se } } } else { - TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); + TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } + peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved,network); } catch ( ... ) { TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -575,7 +579,8 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID))); + const uint64_t nwid = at(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(nwid)); if (network) { if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; @@ -590,7 +595,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

isAllowed(peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,network); return true; } @@ -602,37 +607,38 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay return true; } switch (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { case 1: - if (from != MAC(peer->address(),network->id())) { + if (from != MAC(peer->address(),nwid)) { if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay return true; } } // fall through -- 2 means accept regardless of bridging checks or other restrictions case 2: - RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); + RR->node->putFrame(nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; } - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); } } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,NULL_NETWORK); } } catch ( ... ) { TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -651,7 +657,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),RR->node->now()); - peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -670,7 +676,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared RR->mc->add(now,nwid,group,peer->address()); } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -719,7 +725,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -817,7 +823,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk,NULL_NETWORK); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -839,7 +845,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons network->requestConfiguration(); } else { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false,NULL_NETWORK); return true; } @@ -851,7 +857,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -902,7 +908,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -932,7 +938,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // that cert might be what we needed. if (!network->isAllowed(peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,network); return true; } @@ -959,28 +965,28 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay return true; } - if (from != MAC(peer->address(),network->id())) { + if (from != MAC(peer->address(),nwid)) { if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay return true; } } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { - RR->node->putFrame(network->id(),network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); + RR->node->putFrame(nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); } } @@ -998,9 +1004,9 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); } else { - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,NULL_NETWORK); } } catch ( ... ) { TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -1016,7 +1022,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha // First, subject this to a rate limit if (!peer->shouldRespondToDirectPathPush(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,NULL_NETWORK); return true; } @@ -1079,7 +1085,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha ptr += addrLen; } - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -1123,7 +1129,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); return true; } vlf += signatureLength; @@ -1140,12 +1146,12 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt SharedPtr network(RR->node->network(originatorCredentialNetworkId)); if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); return true; } } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); return true; } @@ -1216,7 +1222,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt } } - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -1261,7 +1267,8 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S } RR->node->postCircuitTestReport(&report); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false,NULL_NETWORK); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -1322,7 +1329,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const break; } - peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false,NULL_NETWORK); } else { TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_path->address().toString().c_str()); } diff --git a/node/Membership.cpp b/node/Membership.cpp index e809e2bd..74a01350 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -89,21 +89,26 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } } -int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) +int Membership::addCredential(const RuntimeEnvironment *RR,const Network *network,const CertificateOfMembership &com) { if (_com == com) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); + sendCredentialsIfNeeded(RR,RR->node->now(),com.issuedTo(),network->config(),(const Capability *)0); return 0; } + const int vr = com.verify(RR); + if (vr == 0) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); if (com.timestamp().first > _com.timestamp().first) { _com = com; } + sendCredentialsIfNeeded(RR,RR->node->now(),com.issuedTo(),network->config(),(const Capability *)0); } else { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); } + return vr; } diff --git a/node/Membership.hpp b/node/Membership.hpp index 293fa4d8..324f92a6 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -31,15 +31,10 @@ #include "Hashtable.hpp" #include "NetworkConfig.hpp" -// Expiration time for capability and tag cache -#define ZT_MEMBERSHIP_STATE_EXPIRATION_TIME ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA - -// Expiration time for Memberships (used in Network::clean()) -#define ZT_MEMBERSHIP_EXPIRATION_TIME ZT_PEER_IN_MEMORY_EXPIRATION - namespace ZeroTier { class RuntimeEnvironment; +class Network; /** * A container for certificates of membership and other network credentials @@ -107,6 +102,7 @@ public: friend class CapabilityIterator; Membership() : + _lastUpdatedMulticast(0), _lastPushAttempt(0), _lastPushedCom(0), _blacklistBefore(0), @@ -130,6 +126,21 @@ public: */ void sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); + /** + * Check whether we should push MULTICAST_LIKEs to this peer + * + * @param now Current time + * @return True if we should update multicasts + */ + inline bool shouldLikeMulticasts(const uint64_t now) const { return ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD); } + + /** + * Set time we last updated multicasts for this peer + * + * @param now Current time + */ + inline void likingMulticasts(const uint64_t now) { _lastUpdatedMulticast = now; } + /** * @param nconf Our network config * @return True if this peer is allowed on this network at all @@ -206,9 +217,12 @@ public: /** * Validate and add a credential if signature is okay and it's otherwise good * + * @param RR Runtime environment + * @param network Network that owns this Membership + * @param com Certificate of membership * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com); + int addCredential(const RuntimeEnvironment *RR,const Network *network,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good @@ -237,20 +251,15 @@ public: /** * Clean up old or stale entries * - * @return Time of most recent activity in this Membership + * @param nconf Network config */ - inline uint64_t clean(const uint64_t now) + inline void clean(const NetworkConfig &nconf) { - uint64_t lastAct = _lastPushedCom; - for(std::map::iterator i(_caps.begin());i!=_caps.end();) { - const uint64_t la = std::max(i->second.lastPushed,i->second.lastReceived); - if ((now - la) > ZT_MEMBERSHIP_STATE_EXPIRATION_TIME) { + if (!isCredentialTimestampValid(nconf,i->second.cap)) { _caps.erase(i++); } else { ++i; - if (la > lastAct) - lastAct = la; } } @@ -258,17 +267,15 @@ public: TState *ts = (TState *)0; Hashtable::Iterator tsi(_tags); while (tsi.next(i,ts)) { - const uint64_t la = std::max(ts->lastPushed,ts->lastReceived); - if ((now - la) > ZT_MEMBERSHIP_STATE_EXPIRATION_TIME) + if (!isCredentialTimestampValid(nconf,ts->tag)) _tags.erase(*i); - else if (la > lastAct) - lastAct = la; } - - return lastAct; } private: + // Last time we pushed MULTICAST_LIKE(s) + uint64_t _lastUpdatedMulticast; + // Last time we checked if credential push was needed uint64_t _lastPushAttempt; diff --git a/node/Network.cpp b/node/Network.cpp index b18a3b22..7c2a4084 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -577,6 +577,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : RR(renv), _uPtr(uptr), _id(nwid), + _lastAnnouncedMulticastGroupsUpstream(0), _mac(renv->identity.address(),nwid), _portInitialized(false), _inboundConfigPacketId(0), @@ -872,8 +873,8 @@ void Network::multicastSubscribe(const MulticastGroup &mg) return; _myMulticastGroups.push_back(mg); std::sort(_myMulticastGroups.begin(),_myMulticastGroups.end()); + _announceMulticastGroups(&mg); } - _announceMulticastGroups(); } void Network::multicastUnsubscribe(const MulticastGroup &mg) @@ -888,20 +889,6 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) _myMulticastGroups.swap(nmg); } -bool Network::tryAnnounceMulticastGroupsTo(const SharedPtr &peer) -{ - Mutex::Lock _l(_lock); - if ( - (_isAllowed(peer)) || - (peer->address() == this->controller()) || - (RR->topology->isUpstream(peer->identity())) - ) { - _announceMulticastGroupsTo(peer,_allMulticastGroups()); - return true; - } - return false; -} - bool Network::applyConfiguration(const NetworkConfig &conf) { if (_destroyed) // sanity check @@ -1094,8 +1081,9 @@ void Network::clean() Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - if ((now - m->clean(now)) > ZT_MEMBERSHIP_EXPIRATION_TIME) - _memberships.erase(*a); + if (RR->topology->getPeerNoCache(*a)) + m->clean(_config); + else _memberships.erase(*a); } } } @@ -1143,7 +1131,7 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); _multicastGroupsBehindMe.set(mg,now); if (tmp != _multicastGroupsBehindMe.size()) - _announceMulticastGroups(); + _announceMulticastGroups(&mg); } void Network::destroy() @@ -1223,61 +1211,66 @@ bool Network::_isAllowed(const SharedPtr &peer) const return false; } -class _MulticastAnnounceAll +void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis) { -public: - _MulticastAnnounceAll(const RuntimeEnvironment *renv,Network *nw) : - _now(renv->node->now()), - _controller(nw->controller()), - _network(nw), - _anchors(nw->config().anchors()), - _upstreamAddresses(renv->topology->upstreamAddresses()) - {} - inline void operator()(Topology &t,const SharedPtr &p) - { - if ( (_network->_isAllowed(p)) || // FIXME: this causes multicast LIKEs for public networks to get spammed, which isn't terrible but is a bit stupid - (p->address() == _controller) || - (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),p->address()) != _upstreamAddresses.end()) || - (std::find(_anchors.begin(),_anchors.end(),p->address()) != _anchors.end()) ) { - peers.push_back(p); + // Assumes _lock is locked + const uint64_t now = RR->node->now(); + + std::vector groups; + if (onlyThis) + groups.push_back(*onlyThis); + else groups = _allMulticastGroups(); + + if ((onlyThis)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) { + if (!onlyThis) + _lastAnnouncedMulticastGroupsUpstream = now; + + // Announce multicast groups to upstream peers (roots, etc.) and also send + // them our COM so that MULTICAST_GATHER can be authenticated properly. + const std::vector

upstreams(RR->topology->upstreamAddresses()); + for(std::vector
::const_iterator a(upstreams.begin());a!=upstreams.end();++a) { + if ((_config.isPrivate())&&(_config.com)) { + Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + _config.com.serialize(outp); + outp.append((uint8_t)0x00); + RR->sw->send(outp,true); + } + _announceMulticastGroupsTo(*a,groups); } } - std::vector< SharedPtr > peers; -private: - const uint64_t _now; - const Address _controller; - Network *const _network; - const std::vector
_anchors; - const std::vector
_upstreamAddresses; -}; -void Network::_announceMulticastGroups() -{ - // Assumes _lock is locked - std::vector allMulticastGroups(_allMulticastGroups()); - _MulticastAnnounceAll gpfunc(RR,this); - RR->topology->eachPeer<_MulticastAnnounceAll &>(gpfunc); - for(std::vector< SharedPtr >::const_iterator i(gpfunc.peers.begin());i!=gpfunc.peers.end();++i) - _announceMulticastGroupsTo(*i,allMulticastGroups); -} -void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std::vector &allMulticastGroups) -{ - // Assumes _lock is locked + // Make sure that all "network anchors" have Membership records so we will + // push multicasts to them. + const std::vector
anchors(_config.anchors()); + for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) + _memberships[*a]; - // Anyone we announce multicast groups to will need our COM to authenticate GATHER requests. + // Send MULTICAST_LIKE(s) to all members of this network { - Membership *m = _memberships.get(peer->address()); - if (m) - m->sendCredentialsIfNeeded(RR,RR->node->now(),peer->address(),_config,(const Capability *)0); + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) { + if ((onlyThis)||(m->shouldLikeMulticasts(now))) { + if (!onlyThis) + m->likingMulticasts(now); + m->sendCredentialsIfNeeded(RR,RR->node->now(),*a,_config,(const Capability *)0); + _announceMulticastGroupsTo(*a,groups); + } + } } +} - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); +void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups) +{ + // Assumes _lock is locked + Packet outp(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); for(std::vector::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { if ((outp.size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { outp.compress(); RR->sw->send(outp,true); - outp.reset(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE); + outp.reset(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); } // network ID, MAC, ADI @@ -1295,7 +1288,6 @@ void Network::_announceMulticastGroupsTo(const SharedPtr &peer,const std:: std::vector Network::_allMulticastGroups() const { // Assumes _lock is locked - std::vector mgs; mgs.reserve(_myMulticastGroups.size() + _multicastGroupsBehindMe.size() + 1); mgs.insert(mgs.end(),_myMulticastGroups.begin(),_myMulticastGroups.end()); @@ -1304,7 +1296,6 @@ std::vector Network::_allMulticastGroups() const mgs.push_back(Network::BROADCAST); std::sort(mgs.begin(),mgs.end()); mgs.erase(std::unique(mgs.begin(),mgs.end()),mgs.end()); - return mgs; } diff --git a/node/Network.hpp b/node/Network.hpp index 45a51bf2..4d0e25b7 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -190,14 +190,6 @@ public: */ void multicastUnsubscribe(const MulticastGroup &mg); - /** - * Announce multicast groups to a peer if that peer is authorized on this network - * - * @param peer Peer to try to announce multicast groups to - * @return True if peer was authorized and groups were announced - */ - bool tryAnnounceMulticastGroupsTo(const SharedPtr &peer); - /** * Apply a NetworkConfig to this network * @@ -272,6 +264,15 @@ public: */ void clean(); + /** + * Announce multicast groups to all members, anchors, etc. + */ + inline void announceMulticastGroups() + { + Mutex::Lock _l(_lock); + _announceMulticastGroups((const MulticastGroup *)0); + } + /** * @return Time of last updated configuration or 0 if none */ @@ -298,23 +299,10 @@ public: /** * Get current network config * - * This returns a const reference to the network config in place, which is safe - * to concurrently access but *may* change during access. Normally this isn't a - * problem, but if it is use configCopy(). - * * @return Network configuration (may be a null config if we don't have one yet) */ inline const NetworkConfig &config() const { return _config; } - /** - * @return A thread-safe copy of our NetworkConfig instead of a const reference - */ - inline NetworkConfig configCopy() const - { - Mutex::Lock _l(_lock); - return _config; - } - /** * @return True if this network has a valid config */ @@ -323,7 +311,7 @@ public: /** * @return Ethernet MAC address for this network's local interface */ - inline const MAC &mac() const throw() { return _mac; } + inline const MAC &mac() const { return _mac; } /** * Find the node on this network that has this MAC behind it (if any) @@ -365,7 +353,7 @@ public: if (com.networkId() != _id) return -1; Mutex::Lock _l(_lock); - return _memberships[com.issuedTo()].addCredential(RR,com); + return _memberships[com.issuedTo()].addCredential(RR,this,com); } /** @@ -417,24 +405,18 @@ public: */ inline void **userPtr() throw() { return &_uPtr; } - inline bool operator==(const Network &n) const throw() { return (_id == n._id); } - inline bool operator!=(const Network &n) const throw() { return (_id != n._id); } - inline bool operator<(const Network &n) const throw() { return (_id < n._id); } - inline bool operator>(const Network &n) const throw() { return (_id > n._id); } - inline bool operator<=(const Network &n) const throw() { return (_id <= n._id); } - inline bool operator>=(const Network &n) const throw() { return (_id >= n._id); } - private: ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _isAllowed(const SharedPtr &peer) const; - void _announceMulticastGroups(); - void _announceMulticastGroupsTo(const SharedPtr &peer,const std::vector &allMulticastGroups); + void _announceMulticastGroups(const MulticastGroup *const onlyThis); + void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; const RuntimeEnvironment *RR; void *_uPtr; uint64_t _id; + uint64_t _lastAnnouncedMulticastGroupsUpstream; MAC _mac; // local MAC address volatile bool _portInitialized; diff --git a/node/Node.cpp b/node/Node.cpp index edd48575..233ddc02 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -261,13 +261,11 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB { Mutex::Lock _l(_networks_m); for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { - if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) { + if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) needConfig.push_back(n->second); - } + n->second->announceMulticastGroups(); } } - - // Request updated configuration for networks that need it for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) (*n)->requestConfiguration(); diff --git a/node/Peer.cpp b/node/Peer.cpp index 3d3ca247..43daeb13 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -45,7 +45,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastReceive(0), _lastUnicastFrame(0), _lastMulticastFrame(0), - _lastAnnouncedTo(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), RR(renv), @@ -66,12 +65,13 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident void Peer::received( const SharedPtr &path, - unsigned int hops, - uint64_t packetId, - Packet::Verb verb, - uint64_t inRePacketId, - Packet::Verb inReVerb, - const bool trustEstablished) + const unsigned int hops, + const uint64_t packetId, + const Packet::Verb verb, + const uint64_t inRePacketId, + const Packet::Verb inReVerb, + const bool trustEstablished, + const SharedPtr &network) { const uint64_t now = RR->node->now(); @@ -197,13 +197,6 @@ void Peer::received( // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) _pushDirectPaths(path,now); } - - if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) { - _lastAnnouncedTo = now; - const std::vector< SharedPtr > networks(RR->node->allNetworks()); - for(std::vector< SharedPtr >::const_iterator n(networks.begin());n!=networks.end();++n) - (*n)->tryAnnounceMulticastGroupsTo(SharedPtr(this)); - } } bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const diff --git a/node/Peer.hpp b/node/Peer.hpp index 7a7453ae..19767a70 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -45,6 +45,8 @@ namespace ZeroTier { +class Network; + /** * Peer on P2P Network (virtual layer 1) */ @@ -103,15 +105,17 @@ public: * @param inRePacketId Packet ID in reply to (default: none) * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP) * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established + * @param network Network to which this packet pertains or NULL if none */ void received( const SharedPtr &path, - unsigned int hops, - uint64_t packetId, - Packet::Verb verb, - uint64_t inRePacketId, - Packet::Verb inReVerb, - const bool trustEstablished); + const unsigned int hops, + const uint64_t packetId, + const Packet::Verb verb, + const uint64_t inRePacketId, + const Packet::Verb inReVerb, + const bool trustEstablished, + const SharedPtr &network); /** * @param now Current time @@ -407,13 +411,12 @@ private: return s; } - unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; + uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; uint8_t _remoteClusterOptimal6[16]; uint64_t _lastUsed; uint64_t _lastReceive; // direct or indirect uint64_t _lastUnicastFrame; uint64_t _lastMulticastFrame; - uint64_t _lastAnnouncedTo; uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; const RuntimeEnvironment *RR; -- cgit v1.2.3 From c7a4da3dd3bb429b5d2f69373388b3b22a6544cb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 15:24:53 -0700 Subject: Turns out we do not need to pass network to receive(). --- node/IncomingPacket.cpp | 70 ++++++++++++++++++++++++------------------------- node/Peer.cpp | 3 +-- node/Peer.hpp | 6 +---- 3 files changed, 36 insertions(+), 43 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 97e1abe3..39f077ff 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -42,8 +42,6 @@ namespace ZeroTier { -static const SharedPtr NULL_NETWORK; - bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) { const Address sourceAddress(source()); @@ -90,7 +88,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; case Packet::VERB_HELLO: return _doHELLO(RR,peer); @@ -174,7 +172,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -341,7 +339,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer _path->send(RR,outp.data(),outp.size(),RR->node->now()); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -463,7 +461,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -507,7 +505,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -539,7 +537,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< } else { TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } - peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -569,7 +567,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr } else { TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved,network); + peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved); } catch ( ... ) { TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -595,7 +593,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

isAllowed(peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,network); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -607,7 +605,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -618,13 +616,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

learnBridgeRoute(from,peer->address()); } else { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } @@ -634,11 +632,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,network); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -657,7 +655,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),RR->node->now()); - peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -676,7 +674,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared RR->mc->add(now,nwid,group,peer->address()); } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -725,7 +723,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -823,7 +821,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk,NULL_NETWORK); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -845,7 +843,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons network->requestConfiguration(); } else { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); return true; } @@ -857,7 +855,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -908,7 +906,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -938,7 +936,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // that cert might be what we needed. if (!network->isAllowed(peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,network); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -965,12 +963,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -979,7 +977,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); // trustEstablished because COM is okay + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } @@ -1004,9 +1002,9 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,network); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); } else { - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -1022,7 +1020,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha // First, subject this to a rate limit if (!peer->shouldRespondToDirectPathPush(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1085,7 +1083,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha ptr += addrLen; } - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -1129,7 +1127,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } vlf += signatureLength; @@ -1146,12 +1144,12 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt SharedPtr network(RR->node->network(originatorCredentialNetworkId)); if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } @@ -1222,7 +1220,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt } } - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -1268,7 +1266,7 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S RR->node->postCircuitTestReport(&report); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -1329,7 +1327,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const break; } - peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false,NULL_NETWORK); + peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false); } else { TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_path->address().toString().c_str()); } diff --git a/node/Peer.cpp b/node/Peer.cpp index 43daeb13..58faab3b 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -70,8 +70,7 @@ void Peer::received( const Packet::Verb verb, const uint64_t inRePacketId, const Packet::Verb inReVerb, - const bool trustEstablished, - const SharedPtr &network) + const bool trustEstablished) { const uint64_t now = RR->node->now(); diff --git a/node/Peer.hpp b/node/Peer.hpp index 19767a70..2e64fb4d 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -45,8 +45,6 @@ namespace ZeroTier { -class Network; - /** * Peer on P2P Network (virtual layer 1) */ @@ -105,7 +103,6 @@ public: * @param inRePacketId Packet ID in reply to (default: none) * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP) * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established - * @param network Network to which this packet pertains or NULL if none */ void received( const SharedPtr &path, @@ -114,8 +111,7 @@ public: const Packet::Verb verb, const uint64_t inRePacketId, const Packet::Verb inReVerb, - const bool trustEstablished, - const SharedPtr &network); + const bool trustEstablished); /** * @param now Current time -- cgit v1.2.3 From 20278bb9e47ec0cc16619d281224473f90f7b048 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 15:34:34 -0700 Subject: Also send MULTICAST_LIKEs to controllers. --- node/Network.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 7c2a4084..4fd88f67 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1237,6 +1237,13 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis) } _announceMulticastGroupsTo(*a,groups); } + + // Announce to controller, which does not need our COM since it obviously + // knows if we are a member. Of course if we already did or are going to + // below then we can skip it here. + const Address c(controller()); + if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) ) + _announceMulticastGroupsTo(c,groups); } // Make sure that all "network anchors" have Membership records so we will -- cgit v1.2.3 From daf8a66ced4ce2bf48ec005d915899d888458b06 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 7 Sep 2016 15:47:20 -0700 Subject: More correct and efficient to initialize member relationship push stuff lazily when member is learned. --- node/Membership.cpp | 4 +--- node/Membership.hpp | 3 +-- node/Network.cpp | 31 +++++++++++++++++++++++-------- node/Network.hpp | 9 +++++---- 4 files changed, 30 insertions(+), 17 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index 74a01350..25ae1d9c 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -89,11 +89,10 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } } -int Membership::addCredential(const RuntimeEnvironment *RR,const Network *network,const CertificateOfMembership &com) +int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) { if (_com == com) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); - sendCredentialsIfNeeded(RR,RR->node->now(),com.issuedTo(),network->config(),(const Capability *)0); return 0; } @@ -104,7 +103,6 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const Network *networ if (com.timestamp().first > _com.timestamp().first) { _com = com; } - sendCredentialsIfNeeded(RR,RR->node->now(),com.issuedTo(),network->config(),(const Capability *)0); } else { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); } diff --git a/node/Membership.hpp b/node/Membership.hpp index 324f92a6..22910148 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -218,11 +218,10 @@ public: * Validate and add a credential if signature is okay and it's otherwise good * * @param RR Runtime environment - * @param network Network that owns this Membership * @param com Certificate of membership * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const Network *network,const CertificateOfMembership &com); + int addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good diff --git a/node/Network.cpp b/node/Network.cpp index 4fd88f67..8b22c097 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -678,7 +678,7 @@ bool Network::filterOutgoingPacket( accept = true; if ((!noTee)&&(cc2)) { - _memberships[cc2].sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,relevantCap); + _membership(cc2).sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,relevantCap); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -710,7 +710,7 @@ bool Network::filterOutgoingPacket( if (accept) { if ((!noTee)&&(cc)) { - _memberships[cc].sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,relevantCap); + _membership(cc).sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,relevantCap); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -724,7 +724,7 @@ bool Network::filterOutgoingPacket( } if ((ztDest != ztDest2)&&(ztDest2)) { - _memberships[ztDest2].sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); + _membership(ztDest2).sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -764,7 +764,7 @@ int Network::filterIncomingPacket( Mutex::Lock _l(_lock); - Membership &m = _memberships[ztDest]; + Membership &m = _membership(ztDest); const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { @@ -791,7 +791,7 @@ int Network::filterIncomingPacket( if (accept) { if (cc2) { - _memberships[cc2].sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,(const Capability *)0); + _membership(cc2).sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,(const Capability *)0); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -822,7 +822,7 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { - _memberships[cc].sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,(const Capability *)0); + _membership(cc).sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,(const Capability *)0); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -836,7 +836,7 @@ int Network::filterIncomingPacket( } if ((ztDest != ztDest2)&&(ztDest2)) { - _memberships[ztDest2].sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,(const Capability *)0); + _membership(ztDest2).sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,(const Capability *)0); Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -1247,7 +1247,8 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis) } // Make sure that all "network anchors" have Membership records so we will - // push multicasts to them. + // push multicasts to them. Note that _membership() also does this but in a + // piecemeal on-demand fashion. const std::vector

anchors(_config.anchors()); for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) _memberships[*a]; @@ -1306,4 +1307,18 @@ std::vector Network::_allMulticastGroups() const return mgs; } +Membership &Network::_membership(const Address &a) +{ + // assumes _lock is locked + const unsigned long ms = _memberships.size(); + Membership &m = _memberships[a]; + if (ms != _memberships.size()) { + const uint64_t now = RR->node->now(); + m.sendCredentialsIfNeeded(RR,now,a,_config,(const Capability *)0); + _announceMulticastGroupsTo(a,_allMulticastGroups()); + m.likingMulticasts(now); + } + return m; +} + } // namespace ZeroTier diff --git a/node/Network.hpp b/node/Network.hpp index 4d0e25b7..bcef2872 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -353,7 +353,7 @@ public: if (com.networkId() != _id) return -1; Mutex::Lock _l(_lock); - return _memberships[com.issuedTo()].addCredential(RR,this,com); + return _membership(com.issuedTo()).addCredential(RR,com); } /** @@ -365,7 +365,7 @@ public: if (cap.networkId() != _id) return -1; Mutex::Lock _l(_lock); - return _memberships[cap.issuedTo()].addCredential(RR,cap); + return _membership(cap.issuedTo()).addCredential(RR,cap); } /** @@ -377,7 +377,7 @@ public: if (tag.networkId() != _id) return -1; Mutex::Lock _l(_lock); - return _memberships[tag.issuedTo()].addCredential(RR,tag); + return _membership(tag.issuedTo()).addCredential(RR,tag); } /** @@ -388,7 +388,7 @@ public: inline void blacklistBefore(const Address &peerAddress,const uint64_t ts) { Mutex::Lock _l(_lock); - _memberships[peerAddress].blacklistBefore(ts); + _membership(peerAddress).blacklistBefore(ts); } /** @@ -412,6 +412,7 @@ private: void _announceMulticastGroups(const MulticastGroup *const onlyThis); void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; + Membership &_membership(const Address &a); // also lazily sends COM and MULTICAST_LIKE(s) if this is a new member const RuntimeEnvironment *RR; void *_uPtr; -- cgit v1.2.3 From 1f6b13b7fdd8d1b79a754338d6ef4f30fd0d4064 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Sep 2016 16:09:56 -0700 Subject: Fix bug causing null addresses to get in memberships[] hash. --- node/Network.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 8b22c097..7aa2a78b 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -656,8 +656,12 @@ bool Network::filterOutgoingPacket( Mutex::Lock _l(_lock); - Membership &m = _memberships[ztDest]; - const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); + Membership *m = (Membership *)0; + unsigned int remoteTagCount = 0; + if (ztDest) { + m = &(_memberships[ztDest]); + remoteTagCount = m->getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); + } switch(_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { @@ -737,8 +741,8 @@ bool Network::filterOutgoingPacket( RR->sw->send(outp,true); return false; // DROP locally, since we redirected - } else if (ztDest) { - m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,relevantCap); + } else if (m) { + m->sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,relevantCap); } } @@ -764,7 +768,7 @@ int Network::filterIncomingPacket( Mutex::Lock _l(_lock); - Membership &m = _membership(ztDest); + Membership &m = _membership(sourcePeer->address()); const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { -- cgit v1.2.3 From 16df2c33631eeb3e123fefa4febf20f202fd476b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Sep 2016 19:48:05 -0700 Subject: Clean up handling of COMs, network access control, and fix a backward compatiblity issue. --- node/IncomingPacket.cpp | 8 ++--- node/Membership.cpp | 10 +++--- node/Membership.hpp | 15 +++++++++ node/Network.cpp | 81 +++++++++++++++++++++++++++---------------------- node/Network.hpp | 24 +++++++-------- node/Node.cpp | 2 +- node/Packet.cpp | 1 + node/Packet.hpp | 3 ++ 8 files changed, 86 insertions(+), 58 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 39f077ff..ac04ce96 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -552,7 +552,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr bool approved = false; if (network) { if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { - if (!network->isAllowed(peer)) { + if (!network->gate(peer,verb(),packetId())) { TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); } else { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); @@ -591,7 +591,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

addCredential(com); } - if (!network->isAllowed(peer)) { + if (!network->gate(peer,verb(),packetId())) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; @@ -619,7 +619,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } - } else if (to != network->mac()) { + } else if ( (to != network->mac()) && (!to.isMulticast()) ) { if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay @@ -934,7 +934,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share // Check membership after we've read any included COM, since // that cert might be what we needed. - if (!network->isAllowed(peer)) { + if (!network->gate(peer,verb(),packetId())) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; diff --git a/node/Membership.cpp b/node/Membership.cpp index 25ae1d9c..4ca008e3 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -24,13 +24,13 @@ #include "Packet.hpp" #include "Node.hpp" -#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 4) +#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 3) namespace ZeroTier { void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) { - if ((now - _lastPushAttempt) < 1000ULL) + if ((now - _lastPushAttempt) < 2000ULL) return; _lastPushAttempt = now; @@ -99,9 +99,11 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe const int vr = com.verify(RR); if (vr == 0) { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); - if (com.timestamp().first > _com.timestamp().first) { + if (com.timestamp().first >= _com.timestamp().first) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); _com = com; + } else { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED but not used (OK but older than current)",com.issuedTo().toString().c_str(),com.networkId()); } } else { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); diff --git a/node/Membership.hpp b/node/Membership.hpp index 22910148..55355fda 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -154,6 +154,21 @@ public: return nconf.com.agreesWith(_com); } + /** + * @return True if this member has been on this network recently (or network is public) + */ + inline bool recentlyAllowedOnNetwork(const NetworkConfig &nconf) const + { + if (nconf.isPublic()) + return true; + if (_com) { + const uint64_t a = _com.timestamp().first; + const std::pair b(nconf.com.timestamp()); + return ((a <= b.first) ? ((b.first - a) <= ZT_PEER_ACTIVITY_TIMEOUT) : true); + } + return false; + } + /** * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time * diff --git a/node/Network.cpp b/node/Network.cpp index 7aa2a78b..2a5f213c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -877,7 +877,7 @@ void Network::multicastSubscribe(const MulticastGroup &mg) return; _myMulticastGroups.push_back(mg); std::sort(_myMulticastGroups.begin(),_myMulticastGroups.end()); - _announceMulticastGroups(&mg); + _pushStateToMembers(&mg); } } @@ -1062,6 +1062,36 @@ void Network::requestConfiguration() _inboundConfigChunks.clear(); } +bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) +{ + Mutex::Lock _l(_lock); + try { + if (_config) { + Membership &m = _membership(peer->address()); + const bool allow = m.isAllowedOnNetwork(_config); + if (allow) { + const uint64_t now = RR->node->now(); + m.sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); + if (m.shouldLikeMulticasts(now)) { + _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); + m.likingMulticasts(now); + } + } else if (m.recentlyAllowedOnNetwork(_config)) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((uint8_t)verb); + outp.append(packetId); + outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); + outp.append(_id); + RR->sw->send(outp,true); + } + return allow; + } + } catch ( ... ) { + TRACE("gate() check failed for peer %s: unexpected exception",peer->address().toString().c_str()); + } + return false; +} + void Network::clean() { const uint64_t now = RR->node->now(); @@ -1135,7 +1165,7 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); _multicastGroupsBehindMe.set(mg,now); if (tmp != _multicastGroupsBehindMe.size()) - _announceMulticastGroups(&mg); + _pushStateToMembers(&mg); } void Network::destroy() @@ -1200,33 +1230,18 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const } } -bool Network::_isAllowed(const SharedPtr &peer) const -{ - // Assumes _lock is locked - try { - if (_config) { - const Membership *const m = _memberships.get(peer->address()); - if (m) - return m->isAllowedOnNetwork(_config); - } - } catch ( ... ) { - TRACE("isAllowed() check failed for peer %s: unexpected exception",peer->address().toString().c_str()); - } - return false; -} - -void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis) +void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked const uint64_t now = RR->node->now(); std::vector groups; - if (onlyThis) - groups.push_back(*onlyThis); + if (newMulticastGroup) + groups.push_back(*newMulticastGroup); else groups = _allMulticastGroups(); - if ((onlyThis)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) { - if (!onlyThis) + if ((newMulticastGroup)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) { + if (!newMulticastGroup) _lastAnnouncedMulticastGroupsUpstream = now; // Announce multicast groups to upstream peers (roots, etc.) and also send @@ -1255,7 +1270,7 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis) // piecemeal on-demand fashion. const std::vector

anchors(_config.anchors()); for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) - _memberships[*a]; + _membership(*a); // Send MULTICAST_LIKE(s) to all members of this network { @@ -1263,11 +1278,13 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis) Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - if ((onlyThis)||(m->shouldLikeMulticasts(now))) { - if (!onlyThis) - m->likingMulticasts(now); + if ( (m->recentlyAllowedOnNetwork(_config)) || (std::find(anchors.begin(),anchors.end(),*a) != anchors.end()) ) { m->sendCredentialsIfNeeded(RR,RR->node->now(),*a,_config,(const Capability *)0); - _announceMulticastGroupsTo(*a,groups); + if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { + if (!newMulticastGroup) + m->likingMulticasts(now); + _announceMulticastGroupsTo(*a,groups); + } } } } @@ -1314,15 +1331,7 @@ std::vector Network::_allMulticastGroups() const Membership &Network::_membership(const Address &a) { // assumes _lock is locked - const unsigned long ms = _memberships.size(); - Membership &m = _memberships[a]; - if (ms != _memberships.size()) { - const uint64_t now = RR->node->now(); - m.sendCredentialsIfNeeded(RR,now,a,_config,(const Capability *)0); - _announceMulticastGroupsTo(a,_allMulticastGroups()); - m.likingMulticasts(now); - } - return m; + return _memberships[a]; } } // namespace ZeroTier diff --git a/node/Network.hpp b/node/Network.hpp index bcef2872..c80f1cba 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -48,7 +48,6 @@ namespace ZeroTier { class RuntimeEnvironment; class Peer; -class _MulticastAnnounceAll; /** * A virtual LAN @@ -56,7 +55,6 @@ class _MulticastAnnounceAll; class Network : NonCopyable { friend class SharedPtr; - friend class _MulticastAnnounceAll; // internal function object public: /** @@ -250,14 +248,14 @@ public: void requestConfiguration(); /** + * Membership check gate for incoming packets related to this network + * * @param peer Peer to check + * @param verb Packet verb + * @param packetId Packet ID * @return True if peer is allowed to communicate on this network */ - inline bool isAllowed(const SharedPtr &peer) const - { - Mutex::Lock _l(_lock); - return _isAllowed(peer); - } + bool gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); /** * Perform cleanup and possibly save state @@ -265,12 +263,12 @@ public: void clean(); /** - * Announce multicast groups to all members, anchors, etc. + * Push state to members such as multicast group memberships and latest COM (if needed) */ - inline void announceMulticastGroups() + inline void pushStateToMembers() { Mutex::Lock _l(_lock); - _announceMulticastGroups((const MulticastGroup *)0); + _pushStateToMembers((const MulticastGroup *)0); } /** @@ -408,11 +406,11 @@ public: private: ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked - bool _isAllowed(const SharedPtr &peer) const; - void _announceMulticastGroups(const MulticastGroup *const onlyThis); + bool _gate(const SharedPtr &peer); + void _pushStateToMembers(const MulticastGroup *const newMulticastGroup); void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; - Membership &_membership(const Address &a); // also lazily sends COM and MULTICAST_LIKE(s) if this is a new member + Membership &_membership(const Address &a); const RuntimeEnvironment *RR; void *_uPtr; diff --git a/node/Node.cpp b/node/Node.cpp index 233ddc02..415385f7 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -263,7 +263,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) needConfig.push_back(n->second); - n->second->announceMulticastGroups(); + n->second->pushStateToMembers(); } } for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) diff --git a/node/Packet.cpp b/node/Packet.cpp index 9630e5bb..9ab68968 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -62,6 +62,7 @@ const char *Packet::errorString(ErrorCode e) case ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND"; case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION"; case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION"; + case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE"; case ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED"; case ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST"; } diff --git a/node/Packet.hpp b/node/Packet.hpp index 27e289fd..5ead2c3d 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1067,6 +1067,9 @@ public: /* Verb or use case not supported/enabled by this node */ ERROR_UNSUPPORTED_OPERATION = 0x05, + /* Network membership certificate update needed */ + ERROR_NEED_MEMBERSHIP_CERTIFICATE = 0x06, + /* Tried to join network, but you're not a member */ ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */ -- cgit v1.2.3 From 0d4109a9f1f119e336d73039251ad17c0e2a56f4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 9 Sep 2016 08:43:58 -0700 Subject: More refactoring to clean up code, and add a gate function to make sure we do not handle OK packets we did not expect. This hardens up a few potential edge cases around security, since such messages might be used to e.g. pollute a cache and DOS under certain conditions. --- controller/EmbeddedNetworkController.cpp | 2 - include/ZeroTierOne.h | 12 +-- node/IncomingPacket.cpp | 156 ++++++++++++++++++------------- node/Membership.hpp | 6 +- node/Multicaster.cpp | 1 + node/Network.cpp | 10 ++ node/Network.hpp | 6 ++ node/Node.cpp | 3 + node/Node.hpp | 34 +++++++ node/OutboundMulticast.cpp | 1 + node/Packet.hpp | 5 +- node/Peer.cpp | 2 + node/Switch.cpp | 11 +-- 13 files changed, 168 insertions(+), 81 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index cf6bd7c9..79560dcc 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1585,7 +1585,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes "\t\"upstream\": \"%.10llx\"," ZT_EOL_S "\t\"current\": \"%.10llx\"," ZT_EOL_S "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"remoteTimestamp\": %llu," ZT_EOL_S "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S "\t\"flags\": %llu," ZT_EOL_S "\t\"sourcePacketHopCount\": %u," ZT_EOL_S @@ -1606,7 +1605,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes (unsigned long long)report->upstream, (unsigned long long)report->current, (unsigned long long)OSUtils::now(), - (unsigned long long)report->remoteTimestamp, (unsigned long long)report->sourcePacketId, (unsigned long long)report->flags, report->sourcePacketHopCount, diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 0c22ae9d..633db7cf 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -154,6 +154,11 @@ extern "C" { */ #define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 8 +/** + * Circuit test report flag: upstream peer authorized in path (e.g. by network COM) + */ +#define ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH 0x0000000000000001ULL + /** * Maximum number of cluster members (and max member ID plus one) */ @@ -1218,18 +1223,13 @@ typedef struct { */ uint64_t timestamp; - /** - * Timestamp on remote device - */ - uint64_t remoteTimestamp; - /** * 64-bit packet ID of packet received by the reporting device */ uint64_t sourcePacketId; /** - * Flags (currently unused, will be zero) + * Flags */ uint64_t flags; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ac04ce96..c8364415 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -156,6 +156,17 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; + case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { + SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + if ((network)&&(network->recentlyAllowedOnNetwork(peer))) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + network->config().com.serialize(outp); + outp.append((uint8_t)0); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); + } + } break; + case Packet::ERROR_NETWORK_ACCESS_DENIED_: { SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) @@ -163,10 +174,12 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr } break; case Packet::ERROR_UNWANTED_MULTICAST: { - uint64_t nwid = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); - MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); - TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",nwid,peer->address().toString().c_str(),mg.toString().c_str()); - RR->mc->remove(nwid,mg,peer->address()); + SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + if ((network)&&(network->gate(peer,verb(),packetId()))) { + MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); + TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); + RR->mc->remove(network->id(),mg,peer->address()); + } } break; default: break; @@ -352,7 +365,12 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); - //TRACE("%s(%s): OK(%s)",source().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); + if (!RR->node->expectingReplyTo(inRePacketId)) { + TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId()); + return true; + } + + //TRACE("%s(%s): OK(%s)",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); switch(inReVerb) { @@ -424,10 +442,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); - const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); - //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); - const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + SharedPtr network(RR->node->network(nwid)); + if ((network)&&(network->gate(peer,verb(),packetId()))) { + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); + //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); + const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); + RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + } } break; case Packet::VERB_MULTICAST_FRAME: { @@ -437,24 +458,26 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p //TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags); - unsigned int offset = 0; + SharedPtr network(RR->node->network(nwid)); + if (network) { + unsigned int offset = 0; - if ((flags & 0x01) != 0) { // deprecated but still used by older peers - CertificateOfMembership com; - offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); - if (com) { - SharedPtr network(RR->node->network(com.networkId())); - if (network) + if ((flags & 0x01) != 0) { // deprecated but still used by older peers + CertificateOfMembership com; + offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); + if (com) network->addCredential(com); } - } - if ((flags & 0x02) != 0) { - // OK(MULTICAST_FRAME) includes implicit gather results - offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; - unsigned int totalKnown = at(offset); offset += 4; - unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + if (network->gate(peer,verb(),packetId())) { + if ((flags & 0x02) != 0) { + // OK(MULTICAST_FRAME) includes implicit gather results + offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; + unsigned int totalKnown = at(offset); offset += 4; + unsigned int count = at(offset); offset += 2; + RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + } + } } } break; @@ -515,27 +538,29 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr rendezvousWith(RR->topology->getPeer(with)); - if (rendezvousWith) { - const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); - const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; - if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { - const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (!RR->topology->isUpstream(peer->identity())) { - TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); - } else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { - RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); - TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + if (!RR->topology->isUpstream(peer->identity())) { + TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str()); + } else { + const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + const SharedPtr rendezvousWith(RR->topology->getPeer(with)); + if (rendezvousWith) { + const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); + const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; + if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { + const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); + if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { + RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); + TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + } else { + TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + } } else { - TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); + TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str()); } } else { - TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str()); + TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } - } else { - TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { @@ -549,25 +574,25 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr try { const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); - bool approved = false; + bool trustEstablished = false; if (network) { - if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { - if (!network->gate(peer,verb(),packetId())) { - TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - } else { + if (!network->gate(peer,verb(),packetId())) { + TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + } else { + trustEstablished = true; + if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); const MAC sourceMac(peer->address(),nwid); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); - approved = true; // this means approved on the network in general, not this packet per se } } } else { TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); } - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved); + peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -580,23 +605,23 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); if (network) { - if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { - const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; + const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; - unsigned int comLen = 0; - if ((flags & 0x01) != 0) { // deprecated but still used by old peers - CertificateOfMembership com; - comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM); - if (com) - network->addCredential(com); - } + unsigned int comLen = 0; + if ((flags & 0x01) != 0) { // inline COM with EXT_FRAME is deprecated but still used with old peers + CertificateOfMembership com; + comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM); + if (com) + network->addCredential(com); + } - if (!network->gate(peer,verb(),packetId())) { - TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); - return true; - } + if (!network->gate(peer,verb(),packetId())) { + TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + return true; + } + if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE); const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO); const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM); @@ -604,7 +629,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str()); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -1139,6 +1164,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt // Add length of second "additional fields" section. vlf += at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); + uint64_t reportFlags = 0; + // Check credentials (signature already verified) if (originatorCredentialNetworkId) { SharedPtr network(RR->node->network(originatorCredentialNetworkId)); @@ -1147,6 +1174,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } + if (network->gate(peer,verb(),packetId())) + reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); @@ -1188,7 +1217,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED); outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED); outp.append((uint16_t)0); // error code, currently unused - outp.append((uint64_t)0); // flags, currently unused + outp.append((uint64_t)reportFlags); outp.append((uint64_t)packetId()); peer->address().appendTo(outp); outp.append((uint8_t)hops()); @@ -1237,7 +1266,6 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); report.testId = at(ZT_PACKET_IDX_PAYLOAD + 8); report.timestamp = at(ZT_PACKET_IDX_PAYLOAD); - report.remoteTimestamp = at(ZT_PACKET_IDX_PAYLOAD + 16); report.sourcePacketId = at(ZT_PACKET_IDX_PAYLOAD + 44); report.flags = at(ZT_PACKET_IDX_PAYLOAD + 36); report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58 diff --git a/node/Membership.hpp b/node/Membership.hpp index 55355fda..d67c6822 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -163,8 +163,10 @@ public: return true; if (_com) { const uint64_t a = _com.timestamp().first; - const std::pair b(nconf.com.timestamp()); - return ((a <= b.first) ? ((b.first - a) <= ZT_PEER_ACTIVITY_TIMEOUT) : true); + if ((_blacklistBefore)&&(a <= _blacklistBefore)) + return false; + const uint64_t b = nconf.com.timestamp().first; + return ((a <= b) ? ((b - a) <= ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) : true); } return false; } diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index a6bff6aa..36d7d2d0 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -253,6 +253,7 @@ void Multicaster::send( outp.append((uint32_t)gatherLimit); if (com) com->serialize(outp); + RR->node->expectReplyTo(outp.packetId()); RR->sw->send(outp,true); } } diff --git a/node/Network.cpp b/node/Network.cpp index 2a5f213c..710e70dd 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1054,6 +1054,7 @@ void Network::requestConfiguration() } else { outp.append((unsigned char)0,16); } + RR->node->expectReplyTo(outp.packetId()); outp.compress(); RR->sw->send(outp,true); @@ -1092,6 +1093,15 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin return false; } +bool Network::recentlyAllowedOnNetwork(const SharedPtr &peer) const +{ + Mutex::Lock _l(_lock); + const Membership *m = _memberships.get(peer->address()); + if (m) + return m->recentlyAllowedOnNetwork(_config); + return false; +} + void Network::clean() { const uint64_t now = RR->node->now(); diff --git a/node/Network.hpp b/node/Network.hpp index c80f1cba..e8d6e2a5 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -257,6 +257,12 @@ public: */ bool gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + /** + * @param peer Peer to check + * @return True if peer has recently been a valid member of this network + */ + bool recentlyAllowedOnNetwork(const SharedPtr &peer) const; + /** * Perform cleanup and possibly save state */ diff --git a/node/Node.cpp b/node/Node.cpp index 415385f7..e8279c62 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -75,6 +75,9 @@ Node::Node( { _online = false; + memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); + memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); + // Use Salsa20 alone as a high-quality non-crypto PRNG { char foo[32]; diff --git a/node/Node.hpp b/node/Node.hpp index 3c0a5e92..315b5248 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -44,6 +44,10 @@ #define TRACE(f,...) {} #endif +// Bit mask for "expecting reply" hash +#define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 +#define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 + namespace ZeroTier { /** @@ -250,6 +254,33 @@ public: void postCircuitTestReport(const ZT_CircuitTestReport *report); void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); + /** + * Register that we are expecting a reply to a packet ID + * + * @param packetId Packet ID to expect reply to + */ + inline void expectReplyTo(const uint64_t packetId) + { + const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); + _expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = packetId; + } + + /** + * Check whether a given packet ID is something we are expecting a reply to + * + * @param packetId Packet ID to check + * @return True if we're expecting a reply + */ + inline bool expectingReplyTo(const uint64_t packetId) const + { + const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); + for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) { + if (_expectingRepliesTo[bucket][i] == packetId) + return true; + } + return false; + } + private: inline SharedPtr _network(uint64_t nwid) const { @@ -266,6 +297,9 @@ private: void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P + uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1]; + uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; + ZT_DataStoreGetFunction _dataStoreGetFunction; ZT_DataStorePutFunction _dataStorePutFunction; ZT_WirePacketSendFunction _wirePacketSendFunction; diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 33c28f65..6e811581 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -91,6 +91,7 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toA //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr2); + RR->node->expectReplyTo(_packet.packetId()); RR->sw->send(_packet,true); } } diff --git a/node/Packet.hpp b/node/Packet.hpp index 5ead2c3d..2ca73a84 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -965,7 +965,7 @@ public: * <[2] 16-bit reporter OS/platform or 0 if not specified> * <[2] 16-bit reporter architecture or 0 if not specified> * <[2] 16-bit error code (set to 0, currently unused)> - * <[8] 64-bit report flags (set to 0, currently unused)> + * <[8] 64-bit report flags> * <[8] 64-bit packet ID of received CIRCUIT_TEST packet> * <[5] upstream ZeroTier address from which CIRCUIT_TEST was received> * <[1] 8-bit packet hop count of received CIRCUIT_TEST> @@ -980,6 +980,9 @@ public: * <[5] ZeroTier address of next hop> * <[...] current best direct path address, if any, 0 if none> * + * Report flags: + * 0x1 - Upstream peer in circuit test path allowed in path (e.g. network COM valid) + * * Circuit test reports can be sent by hops in a circuit test to report * back results. They should include information about the sender as well * as about the paths to which next hops are being sent. diff --git a/node/Peer.cpp b/node/Peer.cpp index 58faab3b..a7a9fcc3 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -266,6 +266,7 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u atAddress.serialize(outp); outp.append((uint64_t)RR->topology->worldId()); outp.append((uint64_t)RR->topology->worldTimestamp()); + RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,false); // HELLO is sent in the clear RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } @@ -274,6 +275,7 @@ void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &at { if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); + RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,true); RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } else { diff --git a/node/Switch.cpp b/node/Switch.cpp index 21d0b3c9..f2a0d35b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -734,13 +734,12 @@ unsigned long Switch::doTimerTasks(uint64_t now) Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) { - SharedPtr root(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); - if (root) { - Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS); + SharedPtr upstream(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); + if (upstream) { + Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); - outp.armor(root->key(),true); - if (root->sendDirect(outp.data(),outp.size(),RR->node->now(),true)) - return root->address(); + RR->node->expectReplyTo(outp.packetId()); + send(outp,true); } return Address(); } -- cgit v1.2.3 From ef8706995786f26df7bcb9f69b2a332419841964 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 9 Sep 2016 09:32:00 -0700 Subject: Fix gating of multicast GATHER replies since these can come from upstream, etc., and fix an issue with sending ECHO to recheck marginal paths. --- node/IncomingPacket.cpp | 4 ++-- node/Network.cpp | 5 +++++ node/Network.hpp | 5 +++++ node/NetworkConfig.hpp | 13 +++++++++++++ node/Switch.cpp | 7 ++----- 5 files changed, 27 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c8364415..1ce942c9 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -443,7 +443,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); SharedPtr network(RR->node->network(nwid)); - if ((network)&&(network->gate(peer,verb(),packetId()))) { + if ((network)&&(network->gateMulticastGather(peer,verb(),packetId()))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); @@ -469,7 +469,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p network->addCredential(com); } - if (network->gate(peer,verb(),packetId())) { + if (network->gateMulticastGather(peer,verb(),packetId())) { if ((flags & 0x02) != 0) { // OK(MULTICAST_FRAME) includes implicit gather results offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; diff --git a/node/Network.cpp b/node/Network.cpp index 710e70dd..a9b14942 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1093,6 +1093,11 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin return false; } +bool Network::gateMulticastGather(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) +{ + return ( (peer->address() == controller()) || RR->topology->isUpstream(peer->identity()) || gate(peer,verb,packetId) || _config.isAnchor(peer->address()) ); +} + bool Network::recentlyAllowedOnNetwork(const SharedPtr &peer) const { Mutex::Lock _l(_lock); diff --git a/node/Network.hpp b/node/Network.hpp index e8d6e2a5..d80b13b9 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -257,6 +257,11 @@ public: */ bool gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + /** + * Check whether this peer is allowed to provide multicast info for this network + */ + bool gateMulticastGather(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + /** * @param peer Peer to check * @return True if peer has recently been a valid member of this network diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index b5ab9ccb..ad1cafa5 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -285,6 +285,19 @@ public: return r; } + /** + * @param a Address to check + * @return True if address is an anchor + */ + inline bool isAnchor(const Address &a) const + { + for(unsigned int i=0;i viaPath(peer->getBestPath(now,false)); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) { - if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) >> 2,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ECHO); - outp.armor(peer->key(),true); - viaPath->send(RR,outp.data(),outp.size(),now); - } + if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) + peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now); viaPath.zero(); } if (!viaPath) { -- cgit v1.2.3 From ab9afbc749f24f08f25dcf8bd6f4263b97c79bb9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 9 Sep 2016 11:36:10 -0700 Subject: (1) Public networks now get COMs even though they do not gate with them since they will need them to push auth for multicast stuff, (2) added a bunch of rate limit circuit breakers for anti-DOS, (3) cleanup. --- controller/EmbeddedNetworkController.cpp | 12 +-- node/Constants.hpp | 20 ++++ node/IncomingPacket.cpp | 152 ++++++++++++++++++----------- node/IncomingPacket.hpp | 2 +- node/Membership.cpp | 2 +- node/Multicaster.cpp | 80 ++++++++++------ node/Multicaster.hpp | 40 ++++++++ node/Network.cpp | 72 ++++++++------ node/Network.hpp | 20 ++-- node/Node.cpp | 2 +- node/Path.hpp | 15 +++ node/Peer.cpp | 160 +++++++++++++++---------------- node/Peer.hpp | 43 ++++++++- 13 files changed, 393 insertions(+), 227 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 79560dcc..861792ed 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -924,13 +924,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - if (_jB(network["private"],true)) { - CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); - if (com.sign(signingId)) { - nc.com = com; - } else { - return NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } + CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); + if (com.sign(signingId)) { + nc.com = com; + } else { + return NETCONF_QUERY_INTERNAL_SERVER_ERROR; } _writeJson(memberJP,member); diff --git a/node/Constants.hpp b/node/Constants.hpp index a625b480..05cd765a 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -236,6 +236,11 @@ */ #define ZT_MULTICAST_EXPLICIT_GATHER_DELAY (ZT_MULTICAST_LIKE_EXPIRE / 10) +/** + * Expiration for credentials presented for MULTICAST_LIKE or MULTICAST_GATHER (for non-network-members) + */ +#define ZT_MULTICAST_CREDENTIAL_EXPIRATON ZT_MULTICAST_LIKE_EXPIRE + /** * Timeout for outgoing multicasts * @@ -263,6 +268,11 @@ */ #define ZT_PATH_MIN_REACTIVATE_INTERVAL 2500 +/** + * Do not accept HELLOs over a given path more often than this + */ +#define ZT_PATH_HELLO_RATE_LIMIT 1000 + /** * Delay between full-fledge pings of directly connected peers */ @@ -283,6 +293,11 @@ */ #define ZT_PEER_ACTIVITY_TIMEOUT 500000 +/** + * General rate limit timeout for multiple packet types (HELLO, etc.) + */ +#define ZT_PEER_GENERAL_INBOUND_RATE_LIMIT 1000 + /** * Delay between requests for updated network autoconf information * @@ -326,6 +341,11 @@ */ #define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000 +/** + * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound + */ +#define ZT_PEER_GENERAL_RATE_LIMIT 1000 + /** * Maximum number of direct path pushes within cutoff time * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 1ce942c9..7f996dab 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -62,11 +62,8 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { - // A null pointer for peer to _doHELLO() tells it to run its own - // special internal authentication logic. This is done for unencrypted - // HELLOs to learn new identities, etc. - SharedPtr tmp; - return _doHELLO(RR,tmp); + // Only HELLO is allowed in the clear, but will still have a MAC + return _doHELLO(RR,false); } SharedPtr peer(RR->topology->getPeer(sourceAddress)); @@ -91,7 +88,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; - case Packet::VERB_HELLO: return _doHELLO(RR,peer); + case Packet::VERB_HELLO: return _doHELLO(RR,true); case Packet::VERB_ERROR: return _doERROR(RR,peer); case Packet::VERB_OK: return _doOK(RR,peer); case Packet::VERB_WHOIS: return _doWHOIS(RR,peer); @@ -192,16 +189,16 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr return true; } -bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer) +bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated) { - /* Note: this is the only packet ever sent in the clear, and it's also - * the only packet that we authenticate via a different path. Authentication - * occurs here and is based on the validity of the identity and the - * integrity of the packet's MAC, but it must be done after we check - * the identity since HELLO is a mechanism for learning new identities - * in the first place. */ - try { + const uint64_t now = RR->node->now(); + + if (!_path->rateGateHello(now)) { + TRACE("dropped HELLO from %s(%s): rate limiting circuit breaker for HELLO on this path tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + const uint64_t pid = packetId(); const Address fromAddress(source()); const unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION]; @@ -228,20 +225,19 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } } - if (protoVersion < ZT_PROTO_VERSION_MIN) { - TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } if (fromAddress != id.address()) { TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_path->address().toString().c_str()); return true; } + if (protoVersion < ZT_PROTO_VERSION_MIN) { + TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } - if (!peer) { // peer == NULL is the normal case here - peer = RR->topology->getPeer(id.address()); - if (peer) { - // We already have an identity with this address -- check for collisions - + SharedPtr peer(RR->topology->getPeer(id.address())); + if (peer) { + // We already have an identity with this address -- check for collisions + if (!alreadyAuthenticated) { if (peer->identity() != id) { // Identity is different from the one we already have -- address collision @@ -273,31 +269,37 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer // Continue at // VALID } - } else { - // We don't already have an identity with this address -- validate and learn it + } // else continue at // VALID + } else { + // We don't already have an identity with this address -- validate and learn it - // Check identity proof of work - if (!id.locallyValidate()) { - TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } + // Sanity check: this basically can't happen + if (alreadyAuthenticated) { + TRACE("dropped HELLO from %s(%s): somehow already authenticated with unknown peer?",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } - // Check packet integrity and authentication - SharedPtr newPeer(new Peer(RR,RR->identity,id)); - if (!dearmor(newPeer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } - peer = RR->topology->addPeer(newPeer); + // Check identity proof of work + if (!id.locallyValidate()) { + TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } - // Continue at // VALID + // Check packet integrity and authentication + SharedPtr newPeer(new Peer(RR,RR->identity,id)); + if (!dearmor(newPeer->key())) { + TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; } + peer = RR->topology->addPeer(newPeer); - // VALID -- if we made it here, packet passed identity and authenticity checks! + // Continue at // VALID } + // VALID -- if we made it here, packet passed identity and authenticity checks! + if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now()); + RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_HELLO); @@ -349,7 +351,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer } outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); @@ -443,7 +445,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); SharedPtr network(RR->node->network(nwid)); - if ((network)&&(network->gateMulticastGather(peer,verb(),packetId()))) { + if ((network)&&(network->gateMulticastGatherReply(peer,verb(),packetId()))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); @@ -469,7 +471,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p network->addCredential(com); } - if (network->gateMulticastGather(peer,verb(),packetId())) { + if (network->gateMulticastGatherReply(peer,verb(),packetId())) { if ((flags & 0x02) != 0) { // OK(MULTICAST_FRAME) includes implicit gather results offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; @@ -494,6 +496,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { + if (!peer->rateGateInboundWhoisRequest(RR->node->now())) { + TRACE("dropped WHOIS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_WHOIS); outp.append(packetId()); @@ -672,6 +679,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

&peer) { try { + if (!peer->rateGateEchoRequest(RR->node->now())) { + TRACE("dropped ECHO from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + const uint64_t pid = packetId(); Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_ECHO); @@ -680,6 +692,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),RR->node->now()); + peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -692,11 +705,35 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared try { const uint64_t now = RR->node->now(); + uint64_t authOnNetwork[256]; + unsigned int authOnNetworkCount = 0; + SharedPtr network; + // Iterate through 18-byte network,MAC,ADI tuples for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr(ptr); - const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); - RR->mc->add(now,nwid,group,peer->address()); + + bool auth = false; + for(unsigned int i=0;iid() != nwid)) + network = RR->node->network(nwid); + if ( ((network)&&(network->gate(peer,verb(),packetId()))) || RR->mc->cacheAuthorized(peer->address(),nwid,now) ) { + auth = true; + if (authOnNetworkCount < 256) // sanity check, packets can't really be this big + authOnNetwork[authOnNetworkCount++] = nwid; + } + } + + if (auth) { + const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); + RR->mc->add(now,nwid,group,peer->address()); + } } peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); @@ -721,7 +758,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S if (network) { if (network->addCredential(com) == 1) return false; // wait for WHOIS - } + } else RR->mc->addCredential(com,false); } } ++p; // skip trailing 0 after COMs if present @@ -759,22 +796,21 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons { try { const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); - - const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); - const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); - const Dictionary metaData(metaDataBytes,metaDataLength); - const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - bool netconfOk = false; + bool trustEstablished = false; if (RR->localNetworkController) { + const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); + const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); + const Dictionary metaData(metaDataBytes,metaDataLength); + NetworkConfig *netconf = new NetworkConfig(); try { switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) { case NetworkController::NETCONF_QUERY_OK: { - netconfOk = true; + trustEstablished = true; Dictionary *dconf = new Dictionary(); try { if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { @@ -846,7 +882,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,trustEstablished); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -897,21 +933,23 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar //TRACE("<address().toString().c_str(),gatherLimit,nwid,mg.toString().c_str()); + const SharedPtr network(RR->node->network(nwid)); + if ((flags & 0x01) != 0) { try { CertificateOfMembership com; com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); if (com) { - SharedPtr network(RR->node->network(nwid)); if (network) network->addCredential(com); + else RR->mc->addCredential(com,false); } } catch ( ... ) { TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str()); } } - if (gatherLimit) { + if ( ( ((network)&&(network->gate(peer,verb(),packetId()))) || (RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now())) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); outp.append(packetId()); @@ -1043,7 +1081,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha const uint64_t now = RR->node->now(); // First, subject this to a rate limit - if (!peer->shouldRespondToDirectPathPush(now)) { + if (!peer->rateGatePushDirectPaths(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 35438f4f..dbaf67b8 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -136,7 +136,7 @@ private: // These are called internally to handle packet contents once it has // been authenticated, decrypted, decompressed, and classified. bool _doERROR(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doHELLO(const RuntimeEnvironment *RR,SharedPtr &peer); // can be called with NULL peer, while all others cannot + bool _doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated); bool _doOK(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer); diff --git a/node/Membership.cpp b/node/Membership.cpp index 4ca008e3..8c2ba673 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -71,7 +71,7 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); + const bool needCom = ((nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); if ( (needCom) || (appendedCaps) || (appendedTags) ) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (needCom) { diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 36d7d2d0..fc8fa1bd 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -34,8 +34,8 @@ namespace ZeroTier { Multicaster::Multicaster(const RuntimeEnvironment *renv) : RR(renv), - _groups(1024), - _groups_m() + _groups(256), + _gatherAuth(256) { } @@ -244,7 +244,7 @@ void Multicaster::send( } for(unsigned int k=0;kconfig())&&(network->config().isPrivate())) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0; + const CertificateOfMembership *com = (network) ? ((network->config().com) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0; Packet outp(explicitGatherPeers[k],RR->identity.address(),Packet::VERB_MULTICAST_GATHER); outp.append(nwid); outp.append((uint8_t)((com) ? 0x01 : 0x00)); @@ -301,42 +301,62 @@ void Multicaster::send( void Multicaster::clean(uint64_t now) { - Mutex::Lock _l(_groups_m); - - Multicaster::Key *k = (Multicaster::Key *)0; - MulticastGroupStatus *s = (MulticastGroupStatus *)0; - Hashtable::Iterator mm(_groups); - while (mm.next(k,s)) { - for(std::list::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) { - if ((tx->expired(now))||(tx->atLimit())) - s->txQueue.erase(tx++); - else ++tx; - } + { + Mutex::Lock _l(_groups_m); + Multicaster::Key *k = (Multicaster::Key *)0; + MulticastGroupStatus *s = (MulticastGroupStatus *)0; + Hashtable::Iterator mm(_groups); + while (mm.next(k,s)) { + for(std::list::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) { + if ((tx->expired(now))||(tx->atLimit())) + s->txQueue.erase(tx++); + else ++tx; + } - unsigned long count = 0; - { - std::vector::iterator reader(s->members.begin()); - std::vector::iterator writer(reader); - while (reader != s->members.end()) { - if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) { - *writer = *reader; - ++writer; - ++count; + unsigned long count = 0; + { + std::vector::iterator reader(s->members.begin()); + std::vector::iterator writer(reader); + while (reader != s->members.end()) { + if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) { + *writer = *reader; + ++writer; + ++count; + } + ++reader; } - ++reader; + } + + if (count) { + s->members.resize(count); + } else if (s->txQueue.empty()) { + _groups.erase(*k); + } else { + s->members.clear(); } } + } - if (count) { - s->members.resize(count); - } else if (s->txQueue.empty()) { - _groups.erase(*k); - } else { - s->members.clear(); + { + Mutex::Lock _l(_gatherAuth_m); + _GatherAuthKey *k = (_GatherAuthKey *)0; + uint64_t *ts = (uint64_t *)ts; + Hashtable<_GatherAuthKey,uint64_t>::Iterator i(_gatherAuth); + while (i.next(k,ts)) { + if ((now - *ts) >= ZT_MULTICAST_CREDENTIAL_EXPIRATON) + _gatherAuth.erase(*k); } } } +void Multicaster::addCredential(const CertificateOfMembership &com,bool alreadyValidated) +{ + if ((alreadyValidated)||(com.verify(RR) == 0)) { + Mutex::Lock _l(_gatherAuth_m); + _gatherAuth[_GatherAuthKey(com.networkId(),com.issuedTo())] = RR->node->now(); + } +} + void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) { // assumes _groups_m is locked diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 51dabc69..8be3b736 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -179,12 +179,52 @@ public: */ void clean(uint64_t now); + /** + * Add an authorization credential + * + * The Multicaster keeps its own track of when valid credentials of network + * membership are presented. This allows it to control MULTICAST_LIKE + * GATHER authorization for networks this node does not belong to. + * + * @param com Certificate of membership + * @param alreadyValidated If true, COM has already been checked and found to be valid and signed + */ + void addCredential(const CertificateOfMembership &com,bool alreadyValidated); + + /** + * Check authorization for GATHER and LIKE for non-network-members + * + * @param a Address of peer + * @param nwid Network ID + * @param now Current time + * @return True if GATHER and LIKE should be allowed + */ + bool cacheAuthorized(const Address &a,const uint64_t nwid,const uint64_t now) const + { + Mutex::Lock _l(_gatherAuth_m); + const uint64_t *p = _gatherAuth.get(_GatherAuthKey(nwid,a)); + return ((p)&&((now - *p) < ZT_MULTICAST_CREDENTIAL_EXPIRATON)); + } + private: void _add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); const RuntimeEnvironment *RR; + Hashtable _groups; Mutex _groups_m; + + struct _GatherAuthKey + { + _GatherAuthKey() : member(0),networkId(0) {} + _GatherAuthKey(const uint64_t nwid,const Address &a) : member(a.toInt()),networkId(nwid) {} + inline unsigned long hashCode() const { return (member ^ networkId); } + inline bool operator==(const _GatherAuthKey &k) const { return ((member == k.member)&&(networkId == k.networkId)); } + uint64_t member; + uint64_t networkId; + }; + Hashtable< _GatherAuthKey,uint64_t > _gatherAuth; + Mutex _gatherAuth_m; }; } // namespace ZeroTier diff --git a/node/Network.cpp b/node/Network.cpp index a9b14942..146f2962 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -866,31 +866,24 @@ bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBr return true; else if (includeBridgedGroups) return _multicastGroupsBehindMe.contains(mg); - else return false; + return false; } void Network::multicastSubscribe(const MulticastGroup &mg) { - { - Mutex::Lock _l(_lock); - if (std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) - return; - _myMulticastGroups.push_back(mg); - std::sort(_myMulticastGroups.begin(),_myMulticastGroups.end()); - _pushStateToMembers(&mg); + Mutex::Lock _l(_lock); + if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) { + _myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg); + _sendUpdatesToMembers(&mg); } } void Network::multicastUnsubscribe(const MulticastGroup &mg) { Mutex::Lock _l(_lock); - std::vector nmg; - for(std::vector::const_iterator i(_myMulticastGroups.begin());i!=_myMulticastGroups.end();++i) { - if (*i != mg) - nmg.push_back(*i); - } - if (nmg.size() != _myMulticastGroups.size()) - _myMulticastGroups.swap(nmg); + std::vector::iterator i(std::lower_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)); + if ( (i != _myMulticastGroups.end()) && (*i == mg) ) + _myMulticastGroups.erase(i); } bool Network::applyConfiguration(const NetworkConfig &conf) @@ -1054,30 +1047,29 @@ void Network::requestConfiguration() } else { outp.append((unsigned char)0,16); } - RR->node->expectReplyTo(outp.packetId()); - outp.compress(); - RR->sw->send(outp,true); - // Expect replies with this in-re packet ID - _inboundConfigPacketId = outp.packetId(); + RR->node->expectReplyTo(_inboundConfigPacketId = outp.packetId()); _inboundConfigChunks.clear(); + + outp.compress(); + RR->sw->send(outp,true); } bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) { + const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); try { if (_config) { Membership &m = _membership(peer->address()); const bool allow = m.isAllowedOnNetwork(_config); if (allow) { - const uint64_t now = RR->node->now(); m.sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); if (m.shouldLikeMulticasts(now)) { _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); m.likingMulticasts(now); } - } else if (m.recentlyAllowedOnNetwork(_config)) { + } else if (m.recentlyAllowedOnNetwork(_config)&&peer->rateGateRequestCredentials(now)) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)verb); outp.append(packetId); @@ -1093,7 +1085,7 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin return false; } -bool Network::gateMulticastGather(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) +bool Network::gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) { return ( (peer->address() == controller()) || RR->topology->isUpstream(peer->identity()) || gate(peer,verb,packetId) || _config.isAnchor(peer->address()) ); } @@ -1180,7 +1172,22 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); _multicastGroupsBehindMe.set(mg,now); if (tmp != _multicastGroupsBehindMe.size()) - _pushStateToMembers(&mg); + _sendUpdatesToMembers(&mg); +} + +int Network::addCredential(const CertificateOfMembership &com) +{ + if (com.networkId() != _id) + return -1; + const Address a(com.issuedTo()); + Mutex::Lock _l(_lock); + Membership &m = _membership(a); + const int result = m.addCredential(RR,com); + if (result == 0) { + m.sendCredentialsIfNeeded(RR,RR->node->now(),a,_config,(const Capability *)0); + RR->mc->addCredential(com,true); + } + return result; } void Network::destroy() @@ -1245,7 +1252,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const } } -void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) +void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked const uint64_t now = RR->node->now(); @@ -1263,7 +1270,7 @@ void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) // them our COM so that MULTICAST_GATHER can be authenticated properly. const std::vector

upstreams(RR->topology->upstreamAddresses()); for(std::vector
::const_iterator a(upstreams.begin());a!=upstreams.end();++a) { - if ((_config.isPrivate())&&(_config.com)) { + if (_config.com) { Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); _config.com.serialize(outp); outp.append((uint8_t)0x00); @@ -1272,12 +1279,17 @@ void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup) _announceMulticastGroupsTo(*a,groups); } - // Announce to controller, which does not need our COM since it obviously - // knows if we are a member. Of course if we already did or are going to - // below then we can skip it here. + // Also announce to controller, and send COM to simplify and generalize behavior even though in theory it does not need it const Address c(controller()); - if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) ) + if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) ) { + if (_config.com) { + Packet outp(c,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + _config.com.serialize(outp); + outp.append((uint8_t)0x00); + RR->sw->send(outp,true); + } _announceMulticastGroupsTo(c,groups); + } } // Make sure that all "network anchors" have Membership records so we will diff --git a/node/Network.hpp b/node/Network.hpp index d80b13b9..7a4065ff 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -260,7 +260,7 @@ public: /** * Check whether this peer is allowed to provide multicast info for this network */ - bool gateMulticastGather(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + bool gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); /** * @param peer Peer to check @@ -276,10 +276,10 @@ public: /** * Push state to members such as multicast group memberships and latest COM (if needed) */ - inline void pushStateToMembers() + inline void sendUpdatesToMembers() { Mutex::Lock _l(_lock); - _pushStateToMembers((const MulticastGroup *)0); + _sendUpdatesToMembers((const MulticastGroup *)0); } /** @@ -332,9 +332,7 @@ public: { Mutex::Lock _l(_lock); const Address *const br = _remoteBridgeRoutes.get(mac); - if (br) - return *br; - return Address(); + return ((br) ? *br : Address()); } /** @@ -357,13 +355,7 @@ public: * @param com Certificate of membership * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - inline int addCredential(const CertificateOfMembership &com) - { - if (com.networkId() != _id) - return -1; - Mutex::Lock _l(_lock); - return _membership(com.issuedTo()).addCredential(RR,com); - } + int addCredential(const CertificateOfMembership &com); /** * @param cap Capability @@ -418,7 +410,7 @@ private: ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); - void _pushStateToMembers(const MulticastGroup *const newMulticastGroup); + void _sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup); void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; Membership &_membership(const Address &a); diff --git a/node/Node.cpp b/node/Node.cpp index e8279c62..59794854 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -266,7 +266,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) needConfig.push_back(n->second); - n->second->pushStateToMembers(); + n->second->sendUpdatesToMembers(); } } for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) diff --git a/node/Path.hpp b/node/Path.hpp index 27cff645..6278532d 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -104,6 +104,7 @@ public: Path() : _lastOut(0), _lastIn(0), + _lastHello(0), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -113,6 +114,7 @@ public: Path(const InetAddress &localAddress,const InetAddress &addr) : _lastOut(0), _lastIn(0), + _lastHello(0), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -229,9 +231,22 @@ public: */ inline uint64_t lastIn() const { return _lastIn; } + /** + * @return True if we should allow HELLO via this path + */ + inline bool rateGateHello(const uint64_t now) + { + if ((now - _lastHello) >= ZT_PATH_HELLO_RATE_LIMIT) { + _lastHello = now; + return true; + } + return false; + } + private: uint64_t _lastOut; uint64_t _lastIn; + uint64_t _lastHello; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often diff --git a/node/Peer.cpp b/node/Peer.cpp index a7a9fcc3..0e6ef333 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -47,6 +47,9 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastMulticastFrame(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), + _lastCredentialRequestSent(0), + _lastWhoisRequestReceived(0), + _lastEchoRequestReceived(0), RR(renv), _remoteClusterOptimal4(0), _vProto(0), @@ -194,7 +197,80 @@ void Peer::received( } } else if (trustEstablished) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) - _pushDirectPaths(path,now); +#ifdef ZT_ENABLE_CLUSTER + // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection + const bool haveCluster = (RR->cluster); +#else + const bool haveCluster = false; +#endif + if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) { + _lastDirectPathPushSent = now; + + std::vector pathsToPush; + + std::vector dps(RR->node->directPaths()); + for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) + pathsToPush.push_back(*i); + + std::vector sym(RR->sa->getSymmetricNatPredictions()); + for(unsigned long i=0,added=0;inode->prng() % sym.size()]); + if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { + pathsToPush.push_back(tmp); + if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) + break; + } + } + + if (pathsToPush.size() > 0) { +#ifdef ZT_TRACE + std::string ps; + for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { + if (ps.length() > 0) + ps.push_back(','); + ps.append(p->toString()); + } + TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); +#endif + + std::vector::const_iterator p(pathsToPush.begin()); + while (p != pathsToPush.end()) { + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); + outp.addSize(2); // leave room for count + + unsigned int count = 0; + while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { + uint8_t addressType = 4; + switch(p->ss_family) { + case AF_INET: + break; + case AF_INET6: + addressType = 6; + break; + default: // we currently only push IP addresses + ++p; + continue; + } + + outp.append((uint8_t)0); // no flags + outp.append((uint16_t)0); // no extensions + outp.append(addressType); + outp.append((uint8_t)((addressType == 4) ? 6 : 18)); + outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); + outp.append((uint16_t)p->port()); + + ++count; + ++p; + } + + if (count) { + outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); + outp.armor(_key,true); + path->send(RR,outp.data(),outp.size(),now); + } + } + } + } } } @@ -368,86 +444,4 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) v6 = _paths[bestp6].path->address(); } -bool Peer::_pushDirectPaths(const SharedPtr &path,uint64_t now) -{ -#ifdef ZT_ENABLE_CLUSTER - // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection - if (RR->cluster) - return false; -#endif - - if ((now - _lastDirectPathPushSent) < ZT_DIRECT_PATH_PUSH_INTERVAL) - return false; - else _lastDirectPathPushSent = now; - - std::vector pathsToPush; - - std::vector dps(RR->node->directPaths()); - for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) - pathsToPush.push_back(*i); - - std::vector sym(RR->sa->getSymmetricNatPredictions()); - for(unsigned long i=0,added=0;inode->prng() % sym.size()]); - if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { - pathsToPush.push_back(tmp); - if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) - break; - } - } - if (pathsToPush.empty()) - return false; - -#ifdef ZT_TRACE - { - std::string ps; - for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { - if (ps.length() > 0) - ps.push_back(','); - ps.append(p->toString()); - } - TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); - } -#endif - - std::vector::const_iterator p(pathsToPush.begin()); - while (p != pathsToPush.end()) { - Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); - outp.addSize(2); // leave room for count - - unsigned int count = 0; - while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) { - uint8_t addressType = 4; - switch(p->ss_family) { - case AF_INET: - break; - case AF_INET6: - addressType = 6; - break; - default: // we currently only push IP addresses - ++p; - continue; - } - - outp.append((uint8_t)0); // no flags - outp.append((uint16_t)0); // no extensions - outp.append(addressType); - outp.append((uint8_t)((addressType == 4) ? 6 : 18)); - outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16)); - outp.append((uint16_t)p->port()); - - ++count; - ++p; - } - - if (count) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); - outp.armor(_key,true); - path->send(RR,outp.data(),outp.size(),now); - } - } - - return true; -} - } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 2e64fb4d..d714b937 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -348,7 +348,7 @@ public: * @param now Current time * @return True if we should respond */ - inline bool shouldRespondToDirectPathPush(const uint64_t now) + inline bool rateGatePushDirectPaths(const uint64_t now) { if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME) ++_directPathPushCutoffCount; @@ -357,6 +357,42 @@ public: return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT); } + /** + * Rate limit gate for sending of ERROR_NEED_MEMBERSHIP_CERTIFICATE + */ + inline bool rateGateRequestCredentials(const uint64_t now) + { + if ((now - _lastCredentialRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastCredentialRequestSent = now; + return true; + } + return false; + } + + /** + * Rate limit gate for inbound WHOIS requests + */ + inline bool rateGateInboundWhoisRequest(const uint64_t now) + { + if ((now - _lastWhoisRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastWhoisRequestReceived = now; + return true; + } + return false; + } + + /** + * Rate limit gate for inbound ECHO requests + */ + inline bool rateGateEchoRequest(const uint64_t now) + { + if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastEchoRequestReceived = now; + return true; + } + return false; + } + /** * Find a common set of addresses by which two peers can link, if any * @@ -378,8 +414,6 @@ public: } private: - bool _pushDirectPaths(const SharedPtr &path,uint64_t now); - inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const { uint64_t s = ZT_PEER_PING_PERIOD + _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); @@ -415,6 +449,9 @@ private: uint64_t _lastMulticastFrame; uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; + uint64_t _lastCredentialRequestSent; + uint64_t _lastWhoisRequestReceived; + uint64_t _lastEchoRequestReceived; const RuntimeEnvironment *RR; uint32_t _remoteClusterOptimal4; uint16_t _vProto; -- cgit v1.2.3 From debc4c45ee138f7e59ec3adbc031cd6e0b77eae0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 9 Sep 2016 11:45:34 -0700 Subject: Set trust established flag in MULTICAST_GATHER. --- node/IncomingPacket.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 7f996dab..a1458a80 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -949,7 +949,8 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar } } - if ( ( ((network)&&(network->gate(peer,verb(),packetId()))) || (RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now())) ) && (gatherLimit > 0) ) { + const bool trustEstablished = ((network)&&(network->gate(peer,verb(),packetId()))); + if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); outp.append(packetId()); @@ -969,7 +970,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } @@ -995,8 +996,6 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share network->addCredential(com); } - // Check membership after we've read any included COM, since - // that cert might be what we needed. if (!network->gate(peer,verb(),packetId())) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); -- cgit v1.2.3 From ea1da3321a8f95eb2f42b62d805841e2d8379e21 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 12 Sep 2016 15:19:21 -0700 Subject: Rate gate requests for COM. --- node/IncomingPacket.cpp | 13 ++++++++----- node/Peer.cpp | 1 + node/Peer.hpp | 13 +++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index a1458a80..eff87350 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -156,11 +156,14 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->recentlyAllowedOnNetwork(peer))) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - network->config().com.serialize(outp); - outp.append((uint8_t)0); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + const uint64_t now = RR->node->now(); + if (peer->rateGateComRequest(now)) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + network->config().com.serialize(outp); + outp.append((uint8_t)0); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),now); + } } } break; diff --git a/node/Peer.cpp b/node/Peer.cpp index 0e6ef333..f7a21ab1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -50,6 +50,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastCredentialRequestSent(0), _lastWhoisRequestReceived(0), _lastEchoRequestReceived(0), + _lastComRequestReceived(0), RR(renv), _remoteClusterOptimal4(0), _vProto(0), diff --git a/node/Peer.hpp b/node/Peer.hpp index d714b937..a804dd91 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -393,6 +393,18 @@ public: return false; } + /** + * Rate gate requests for network COM + */ + inline bool rateGateComRequest(const uint64_t now) + { + if ((now - _lastComRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastComRequestReceived = now; + return true; + } + return false; + } + /** * Find a common set of addresses by which two peers can link, if any * @@ -452,6 +464,7 @@ private: uint64_t _lastCredentialRequestSent; uint64_t _lastWhoisRequestReceived; uint64_t _lastEchoRequestReceived; + uint64_t _lastComRequestReceived; const RuntimeEnvironment *RR; uint32_t _remoteClusterOptimal4; uint16_t _vProto; -- cgit v1.2.3 From cba37c610786417ad73f455cfb3b6c5d0daf07e8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 13 Sep 2016 10:13:23 -0700 Subject: Add a few more rate limit gates for anti-DOS hardening. --- node/Constants.hpp | 20 +++++++++++++----- node/IncomingPacket.cpp | 54 ++++++++++++++++++++++++++++++++++++++++--------- node/Peer.cpp | 4 +++- node/Peer.hpp | 24 +++++++++++++--------- 4 files changed, 77 insertions(+), 25 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 05cd765a..afd2e4ec 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -341,11 +341,6 @@ */ #define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000 -/** - * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound - */ -#define ZT_PEER_GENERAL_RATE_LIMIT 1000 - /** * Maximum number of direct path pushes within cutoff time * @@ -355,6 +350,21 @@ */ #define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5 +/** + * Time horizon for VERB_NETWORK_CREDENTIALS cutoff + */ +#define ZT_PEER_CREDENTIALS_CUTOFF_TIME 60000 + +/** + * Maximum number of VERB_NETWORK_CREDENTIALS within cutoff time + */ +#define ZT_PEER_CREDEITIALS_CUTOFF_LIMIT 15 + +/** + * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound + */ +#define ZT_PEER_GENERAL_RATE_LIMIT 1000 + /** * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6) */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index eff87350..84503406 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -133,6 +133,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr switch(errorCode) { case Packet::ERROR_OBJ_NOT_FOUND: + // Object not found, currently only meaningful from network controllers. if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) @@ -141,6 +142,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr break; case Packet::ERROR_UNSUPPORTED_OPERATION: + // This can be sent in response to any operation, though right now we only + // consider it meaningful from network controllers. This would indicate + // that the queried node does not support acting as a controller. if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) @@ -149,11 +153,18 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr break; case Packet::ERROR_IDENTITY_COLLISION: + // Roots are the only peers currently permitted to state authoritatively + // that an identity has collided. When this occurs the node should be shut + // down and a new identity created. The odds of this ever happening are + // very low. if (RR->topology->isRoot(peer->identity())) RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { + // This error can be sent in response to any packet that fails network + // authorization. We only listen to it if it's from a peer that has recently + // been authorized on this network. SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->recentlyAllowedOnNetwork(peer))) { const uint64_t now = RR->node->now(); @@ -168,12 +179,15 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr } break; case Packet::ERROR_NETWORK_ACCESS_DENIED_: { + // Network controller: network access denied. SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) network->setAccessDenied(); } break; case Packet::ERROR_UNWANTED_MULTICAST: { + // Members of networks can use this error to indicate that they no longer + // want to receive multicasts on a given channel. SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->gate(peer,verb(),packetId()))) { MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); @@ -301,6 +315,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut // VALID -- if we made it here, packet passed identity and authenticity checks! + // Learn our external surface address from other peers to help us negotiate symmetric NATs + // and detect changes to our global IP that can trigger path renegotiation. if ((externalSurfaceAddress)&&(hops() == 0)) RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); @@ -370,6 +386,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); + // Don't parse OK packets that are not in response to a packet ID we sent if (!RR->node->expectingReplyTo(inRePacketId)) { TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId()); return true; @@ -711,6 +728,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared uint64_t authOnNetwork[256]; unsigned int authOnNetworkCount = 0; SharedPtr network; + bool trustEstablished = false; // Iterate through 18-byte network,MAC,ADI tuples for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptrid() != nwid)) network = RR->node->network(nwid); - if ( ((network)&&(network->gate(peer,verb(),packetId()))) || RR->mc->cacheAuthorized(peer->address(),nwid,now) ) { + const bool authOnNet = ((network)&&(network->gate(peer,verb(),packetId()))); + trustEstablished |= authOnNet; + if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { auth = true; if (authOnNetworkCount < 256) // sanity check, packets can't really be this big authOnNetwork[authOnNetworkCount++] = nwid; @@ -739,7 +759,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -749,9 +769,15 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { + if (!peer->rateGateCredentialsReceived(RR->node->now())) { + TRACE("dropped NETWORK_CREDENTIALS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + return true; + } + CertificateOfMembership com; Capability cap; Tag tag; + bool trustEstablished = false; unsigned int p = ZT_PACKET_IDX_PAYLOAD; while ((p < size())&&((*this)[p])) { @@ -759,8 +785,10 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S if (com) { SharedPtr network(RR->node->network(com.networkId())); if (network) { - if (network->addCredential(com) == 1) - return false; // wait for WHOIS + switch (network->addCredential(com)) { + case 0: trustEstablished = true; break; + case 1: return false; // wait for WHOIS + } } else RR->mc->addCredential(com,false); } } @@ -772,8 +800,10 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += cap.deserialize(*this,p); SharedPtr network(RR->node->network(cap.networkId())); if (network) { - if (network->addCredential(cap) == 1) - return false; // wait for WHOIS + switch (network->addCredential(cap)) { + case 0: trustEstablished = true; break; + case 1: return false; // wait for WHOIS + } } } @@ -782,13 +812,15 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += tag.deserialize(*this,p); SharedPtr network(RR->node->network(tag.networkId())); if (network) { - if (network->addCredential(tag) == 1) - return false; // wait for WHOIS + switch (network->addCredential(tag)) { + case 0: trustEstablished = true; break; + case 1: return false; // wait for WHOIS + } } } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -900,11 +932,13 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons { try { const uint64_t nwid = at(ZT_PACKET_IDX_PAYLOAD); + bool trustEstablished = false; if (Network::controllerFor(nwid) == peer->address()) { SharedPtr network(RR->node->network(nwid)); if (network) { network->requestConfiguration(); + trustEstablished = true; } else { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid); peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); @@ -919,7 +953,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } diff --git a/node/Peer.cpp b/node/Peer.cpp index f7a21ab1..560ca786 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -51,6 +51,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastWhoisRequestReceived(0), _lastEchoRequestReceived(0), _lastComRequestReceived(0), + _lastCredentialsReceived(0), RR(renv), _remoteClusterOptimal4(0), _vProto(0), @@ -60,7 +61,8 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _id(peerIdentity), _numPaths(0), _latency(0), - _directPathPushCutoffCount(0) + _directPathPushCutoffCount(0), + _credentialsCutoffCount(0) { memset(_remoteClusterOptimal6,0,sizeof(_remoteClusterOptimal6)); if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) diff --git a/node/Peer.hpp b/node/Peer.hpp index a804dd91..5382e3f0 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -338,15 +338,7 @@ public: inline bool remoteVersionKnown() const throw() { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); } /** - * Update direct path push stats and return true if we should respond - * - * This is a circuit breaker to make VERB_PUSH_DIRECT_PATHS not particularly - * useful as a DDOS amplification attack vector. Otherwise a malicious peer - * could send loads of these and cause others to bombard arbitrary IPs with - * traffic. - * - * @param now Current time - * @return True if we should respond + * Rate limit gate for VERB_PUSH_DIRECT_PATHS */ inline bool rateGatePushDirectPaths(const uint64_t now) { @@ -357,6 +349,18 @@ public: return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT); } + /** + * Rate limit gate for VERB_NETWORK_CREDENTIALS + */ + inline bool rateGateCredentialsReceived(const uint64_t now) + { + if ((now - _lastCredentialsReceived) <= ZT_PEER_CREDENTIALS_CUTOFF_TIME) + ++_credentialsCutoffCount; + else _credentialsCutoffCount = 0; + _lastCredentialsReceived = now; + return (_directPathPushCutoffCount < ZT_PEER_CREDEITIALS_CUTOFF_LIMIT); + } + /** * Rate limit gate for sending of ERROR_NEED_MEMBERSHIP_CERTIFICATE */ @@ -465,6 +469,7 @@ private: uint64_t _lastWhoisRequestReceived; uint64_t _lastEchoRequestReceived; uint64_t _lastComRequestReceived; + uint64_t _lastCredentialsReceived; const RuntimeEnvironment *RR; uint32_t _remoteClusterOptimal4; uint16_t _vProto; @@ -483,6 +488,7 @@ private: unsigned int _numPaths; unsigned int _latency; unsigned int _directPathPushCutoffCount; + unsigned int _credentialsCutoffCount; AtomicCounter __refCount; }; -- cgit v1.2.3 From 0da9a9a3e01772bf9d534289c755ba96bd099ac9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 13 Sep 2016 10:33:03 -0700 Subject: Set trustEstablished in a few more places. --- node/IncomingPacket.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 84503406..7510fec8 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -385,6 +385,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); + bool trustEstablished = false; // Don't parse OK packets that are not in response to a packet ID we sent if (!RR->node->expectingReplyTo(inRePacketId)) { @@ -446,6 +447,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); if ((network)&&(network->controller() == peer->address())) { + trustEstablished = true; const unsigned int chunkLen = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN); const void *chunkData = field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,chunkLen); unsigned int chunkIndex = 0; @@ -466,6 +468,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); SharedPtr network(RR->node->network(nwid)); if ((network)&&(network->gateMulticastGatherReply(peer,verb(),packetId()))) { + trustEstablished = true; const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); @@ -492,6 +495,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p } if (network->gateMulticastGatherReply(peer,verb(),packetId())) { + trustEstablished = true; if ((flags & 0x02) != 0) { // OK(MULTICAST_FRAME) includes implicit gather results offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; @@ -506,7 +510,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); + peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,trustEstablished); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } -- cgit v1.2.3 From 8ef0e4bbafbd87c32c62553bd84d87bd0eda0e06 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 13 Sep 2016 10:46:36 -0700 Subject: Get rid of HELLO rate gate on path since its basically worthless. There are 65535 ports per IP. --- node/IncomingPacket.cpp | 13 ++++--------- node/Path.hpp | 15 --------------- 2 files changed, 4 insertions(+), 24 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 7510fec8..64dccef3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -211,11 +211,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut try { const uint64_t now = RR->node->now(); - if (!_path->rateGateHello(now)) { - TRACE("dropped HELLO from %s(%s): rate limiting circuit breaker for HELLO on this path tripped",source().toString().c_str(),_path->address().toString().c_str()); - return true; - } - const uint64_t pid = packetId(); const Address fromAddress(source()); const unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION]; @@ -258,14 +253,14 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut if (peer->identity() != id) { // Identity is different from the one we already have -- address collision - unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; + uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { if (dearmor(key)) { // ensure packet is authentic, otherwise drop TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_path->address().toString().c_str()); Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_HELLO); + outp.append((uint8_t)Packet::VERB_HELLO); outp.append((uint64_t)pid); - outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION); + outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION); outp.armor(key,true); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } else { @@ -296,7 +291,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } - // Check identity proof of work + // Check that identity's address is valid as per the derivation function if (!id.locallyValidate()) { TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); return true; diff --git a/node/Path.hpp b/node/Path.hpp index 6278532d..27cff645 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -104,7 +104,6 @@ public: Path() : _lastOut(0), _lastIn(0), - _lastHello(0), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -114,7 +113,6 @@ public: Path(const InetAddress &localAddress,const InetAddress &addr) : _lastOut(0), _lastIn(0), - _lastHello(0), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -231,22 +229,9 @@ public: */ inline uint64_t lastIn() const { return _lastIn; } - /** - * @return True if we should allow HELLO via this path - */ - inline bool rateGateHello(const uint64_t now) - { - if ((now - _lastHello) >= ZT_PATH_HELLO_RATE_LIMIT) { - _lastHello = now; - return true; - } - return false; - } - private: uint64_t _lastOut; uint64_t _lastIn; - uint64_t _lastHello; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often -- cgit v1.2.3 From 5b6d27e65919cf0429feb2d8a9ce0b6164153efd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 13 Sep 2016 14:27:18 -0700 Subject: Implement relay policy, and setting multicast limit to 0 now disables multicast on the network as would be expected. --- include/ZeroTierOne.h | 38 ++++++++++++++++++----- node/Constants.hpp | 9 ++++-- node/IncomingPacket.cpp | 16 ++++++++-- node/Node.cpp | 22 ++++++++++++- node/Node.hpp | 3 ++ node/Path.hpp | 13 ++++++++ node/Peer.cpp | 6 ++++ node/Peer.hpp | 18 +++++++---- node/Switch.cpp | 31 +++++++++++++++++-- osdep/ManagedRoute.cpp | 80 +++++++++++++----------------------------------- osdep/ManagedRoute.hpp | 4 +-- service/ControlPlane.cpp | 2 +- 12 files changed, 159 insertions(+), 83 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 633db7cf..e4ea92b4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -870,19 +870,28 @@ enum ZT_VirtualNetworkConfigOperation ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4 }; +enum ZT_RelayPolicy +{ + ZT_RELAY_POLICY_NEVER = 0, + ZT_RELAY_POLICY_TRUSTED = 1, + ZT_RELAY_POLICY_ALWAYS = 2 +}; + /** * What trust hierarchy role does this peer have? */ -enum ZT_PeerRole { +enum ZT_PeerRole +{ ZT_PEER_ROLE_LEAF = 0, // ordinary node - ZT_PEER_ROLE_RELAY = 1, // relay node - ZT_PEER_ROLE_ROOT = 2 // root server + ZT_PEER_ROLE_UPSTREAM = 1, // upstream node + ZT_PEER_ROLE_ROOT = 2 // global root }; /** * Vendor ID */ -enum ZT_Vendor { +enum ZT_Vendor +{ ZT_VENDOR_UNSPECIFIED = 0, ZT_VENDOR_ZEROTIER = 1 }; @@ -890,7 +899,8 @@ enum ZT_Vendor { /** * Platform type */ -enum ZT_Platform { +enum ZT_Platform +{ ZT_PLATFORM_UNSPECIFIED = 0, ZT_PLATFORM_LINUX = 1, ZT_PLATFORM_WINDOWS = 2, @@ -905,13 +915,15 @@ enum ZT_Platform { ZT_PLATFORM_VXWORKS = 11, ZT_PLATFORM_FREERTOS = 12, ZT_PLATFORM_SYSBIOS = 13, - ZT_PLATFORM_HURD = 14 + ZT_PLATFORM_HURD = 14, + ZT_PLATFORM_WEB = 15 }; /** * Architecture type */ -enum ZT_Architecture { +enum ZT_Architecture +{ ZT_ARCHITECTURE_UNSPECIFIED = 0, ZT_ARCHITECTURE_X86 = 1, ZT_ARCHITECTURE_X64 = 2, @@ -926,7 +938,8 @@ enum ZT_Architecture { ZT_ARCHITECTURE_SPARC32 = 11, ZT_ARCHITECTURE_SPARC64 = 12, ZT_ARCHITECTURE_DOTNET_CLR = 13, - ZT_ARCHITECTURE_JAVA_JVM = 14 + ZT_ARCHITECTURE_JAVA_JVM = 14, + ZT_ARCHITECTURE_WEB = 15 }; /** @@ -1681,6 +1694,15 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( */ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); +/** + * Set node's relay policy + * + * @param node Node instance + * @param rp New relay policy + * @return OK(0) or error code + */ +enum ZT_ResultCode ZT_Node_setRelayPolicy(ZT_Node *node,enum ZT_RelayPolicy rp); + /** * Join a network * diff --git a/node/Constants.hpp b/node/Constants.hpp index afd2e4ec..b3c3dec0 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -350,6 +350,11 @@ */ #define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5 +/** + * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6) + */ +#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 4 + /** * Time horizon for VERB_NETWORK_CREDENTIALS cutoff */ @@ -366,9 +371,9 @@ #define ZT_PEER_GENERAL_RATE_LIMIT 1000 /** - * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6) + * How long is a path or peer considered to have a trust relationship with us (for e.g. relay policy) since last trusted established packet? */ -#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 4 +#define ZT_TRUST_EXPIRATION 600000 /** * Enable support for older network configurations from older (pre-1.1.6) controllers diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 64dccef3..9bc41d47 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -670,8 +670,14 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } - } else if ( (to != network->mac()) && (!to.isMulticast()) ) { - if (!network->config().permitsBridging(RR->identity.address())) { + } else if (to != network->mac()) { + if (to.isMulticast()) { + if (network->config().multicastLimit == 0) { + TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: network %.16llx does not allow multicast",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + return true; + } + } else if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; @@ -1038,6 +1044,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share return true; } + if (network->config().multicastLimit == 0) { + TRACE("dropped MULTICAST_FRAME from %s(%s): network %.16llx does not allow multicast",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + return true; + } + unsigned int gatherLimit = 0; if ((flags & 0x02) != 0) { gatherLimit = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT); diff --git a/node/Node.cpp b/node/Node.cpp index 59794854..51f1b5c0 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -71,7 +71,8 @@ Node::Node( _prngStreamPtr(0), _now(now), _lastPingCheck(0), - _lastHousekeepingRun(0) + _lastHousekeepingRun(0), + _relayPolicy(ZT_RELAY_POLICY_TRUSTED) { _online = false; @@ -118,6 +119,9 @@ Node::Node( throw; } + if (RR->topology->amRoot()) + _relayPolicy = ZT_RELAY_POLICY_ALWAYS; + postEvent(ZT_EVENT_UP); } @@ -131,6 +135,7 @@ Node::~Node() delete RR->topology; delete RR->mc; delete RR->sw; + #ifdef ZT_ENABLE_CLUSTER delete RR->cluster; #endif @@ -319,6 +324,12 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB return ZT_RESULT_OK; } +ZT_ResultCode Node::setRelayPolicy(enum ZT_RelayPolicy rp) +{ + _relayPolicy = rp; + return ZT_RESULT_OK; +} + ZT_ResultCode Node::join(uint64_t nwid,void *uptr) { Mutex::Lock _l(_networks_m); @@ -824,6 +835,15 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol } } +enum ZT_ResultCode ZT_Node_setRelayPolicy(ZT_Node *node,enum ZT_RelayPolicy rp) +{ + try { + return reinterpret_cast(node)->setRelayPolicy(rp); + } catch ( ... ) { + return ZT_RESULT_FATAL_ERROR_INTERNAL; + } +} + enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr) { try { diff --git a/node/Node.hpp b/node/Node.hpp index 315b5248..56869816 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -91,6 +91,7 @@ public: unsigned int frameLength, volatile uint64_t *nextBackgroundTaskDeadline); ZT_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); + ZT_ResultCode setRelayPolicy(enum ZT_RelayPolicy rp); ZT_ResultCode join(uint64_t nwid,void *uptr); ZT_ResultCode leave(uint64_t nwid,void **uptr); ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); @@ -245,6 +246,7 @@ public: inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,nwid,nuptr,op,nc); } inline bool online() const throw() { return _online; } + inline ZT_RelayPolicy relayPolicy() const { return _relayPolicy; } #ifdef ZT_TRACE void postTrace(const char *module,unsigned int line,const char *fmt,...); @@ -326,6 +328,7 @@ private: uint64_t _now; uint64_t _lastPingCheck; uint64_t _lastHousekeepingRun; + ZT_RelayPolicy _relayPolicy; bool _online; }; diff --git a/node/Path.hpp b/node/Path.hpp index 27cff645..5993be69 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -104,6 +104,7 @@ public: Path() : _lastOut(0), _lastIn(0), + _lastTrustEstablishedPacketReceived(0), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -113,6 +114,7 @@ public: Path(const InetAddress &localAddress,const InetAddress &addr) : _lastOut(0), _lastIn(0), + _lastTrustEstablishedPacketReceived(0), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -126,6 +128,11 @@ public: */ inline void received(const uint64_t t) { _lastIn = t; } + /** + * Set time last trusted packet was received (done in Peer::received()) + */ + inline void trustedPacketReceived(const uint64_t t) { _lastTrustEstablishedPacketReceived = t; } + /** * Send a packet via this path (last out time is also updated) * @@ -159,6 +166,11 @@ public: */ inline InetAddress::IpScope ipScope() const { return _ipScope; } + /** + * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms + */ + inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } + /** * @return Preference rank, higher == better */ @@ -232,6 +244,7 @@ public: private: uint64_t _lastOut; uint64_t _lastIn; + uint64_t _lastTrustEstablishedPacketReceived; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often diff --git a/node/Peer.cpp b/node/Peer.cpp index 560ca786..78af9063 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -52,6 +52,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastEchoRequestReceived(0), _lastComRequestReceived(0), _lastCredentialsReceived(0), + _lastTrustEstablishedPacketReceived(0), RR(renv), _remoteClusterOptimal4(0), _vProto(0), @@ -132,6 +133,11 @@ void Peer::received( else if (verb == Packet::VERB_MULTICAST_FRAME) _lastMulticastFrame = now; + if (trustEstablished) { + _lastTrustEstablishedPacketReceived = now; + path->trustedPacketReceived(now); + } + if (hops == 0) { bool pathIsConfirmed = false; { diff --git a/node/Peer.hpp b/node/Peer.hpp index 5382e3f0..1ae239bc 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -312,7 +312,7 @@ public: /** * @return 256-bit secret symmetric encryption key */ - inline const unsigned char *key() const throw() { return _key; } + inline const unsigned char *key() const { return _key; } /** * Set the currently known remote version of this peer's client @@ -330,12 +330,17 @@ public: _vRevision = (uint16_t)vrev; } - inline unsigned int remoteVersionProtocol() const throw() { return _vProto; } - inline unsigned int remoteVersionMajor() const throw() { return _vMajor; } - inline unsigned int remoteVersionMinor() const throw() { return _vMinor; } - inline unsigned int remoteVersionRevision() const throw() { return _vRevision; } + inline unsigned int remoteVersionProtocol() const { return _vProto; } + inline unsigned int remoteVersionMajor() const { return _vMajor; } + inline unsigned int remoteVersionMinor() const { return _vMinor; } + inline unsigned int remoteVersionRevision() const { return _vRevision; } - inline bool remoteVersionKnown() const throw() { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); } + inline bool remoteVersionKnown() const { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); } + + /** + * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms + */ + inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } /** * Rate limit gate for VERB_PUSH_DIRECT_PATHS @@ -470,6 +475,7 @@ private: uint64_t _lastEchoRequestReceived; uint64_t _lastComRequestReceived; uint64_t _lastCredentialsReceived; + uint64_t _lastTrustEstablishedPacketReceived; const RuntimeEnvironment *RR; uint32_t _remoteClusterOptimal4; uint16_t _vProto; diff --git a/node/Switch.cpp b/node/Switch.cpp index ea92c99a..beb36b6c 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -105,7 +105,18 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from const Address destination(fragment.destination()); if (destination != RR->identity.address()) { - // Fragment is not for us, so try to relay it + switch(RR->node->relayPolicy()) { + case ZT_RELAY_POLICY_ALWAYS: + break; + case ZT_RELAY_POLICY_TRUSTED: + if (!path->trustEstablished(now)) + return; + break; + // case ZT_RELAY_POLICY_NEVER: + default: + return; + } + if (fragment.hops() < ZT_RELAY_MAX_HOPS) { fragment.incrementHops(); @@ -203,9 +214,20 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); if (destination != RR->identity.address()) { + switch(RR->node->relayPolicy()) { + case ZT_RELAY_POLICY_ALWAYS: + break; + case ZT_RELAY_POLICY_TRUSTED: + if (!path->trustEstablished(now)) + return; + break; + // case ZT_RELAY_POLICY_NEVER: + default: + return; + } + Packet packet(data,len); - // Packet is not for us, so try to relay it if (packet.hops() < ZT_RELAY_MAX_HOPS) { packet.incrementHops(); @@ -327,6 +349,11 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } if (to.isMulticast()) { + if (network->config().multicastLimit == 0) { + TRACE("%.16llx: dropped multicast: not allowed on network",network->id()); + return; + } + // Destination is a multicast address (including broadcast) MulticastGroup mg(to,0); diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 711a09ed..ae20bb34 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -436,12 +436,12 @@ bool ManagedRoute::sync() } if (!_applied.count(leftt)) { - _applied.insert(leftt); + _applied[rightt] = false; // not ifscoped _routeCmd("add",leftt,_via,(const char *)0,(_via) ? (const char *)0 : _device); _routeCmd("change",leftt,_via,(const char *)0,(_via) ? (const char *)0 : _device); } if ((rightt)&&(!_applied.count(rightt))) { - _applied.insert(rightt); + _applied[rightt] = false; // not ifscoped _routeCmd("add",rightt,_via,(const char *)0,(_via) ? (const char *)0 : _device); _routeCmd("change",rightt,_via,(const char *)0,(_via) ? (const char *)0 : _device); } @@ -457,7 +457,7 @@ bool ManagedRoute::sync() } } else { if (!_applied.count(_target)) { - _applied.insert(_target); + _applied[_target] = true; // ifscoped _routeCmd("add",_target,_via,_device,(_via) ? (const char *)0 : _device); _routeCmd("change",_target,_via,_device,(_via) ? (const char *)0 : _device); } @@ -468,65 +468,27 @@ bool ManagedRoute::sync() #ifdef __LINUX__ // ---------------------------------------------------------- - //if (needBifurcation) { - if (!_applied.count(leftt)) { - _applied.insert(leftt); - _routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device); - } - if ((rightt)&&(!_applied.count(rightt))) { - _applied.insert(rightt); - _routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device); - } - /*if (_applied.count(_target)) { - _applied.erase(_target); - _routeCmd("del",_target,_via,(_via) ? (const char *)0 : _device); - }*/ - /*} else { - if (_applied.count(leftt)) { - _applied.erase(leftt); - _routeCmd("del",leftt,_via,(_via) ? (const char *)0 : _device); - } - if ((rightt)&&(_applied.count(rightt))) { - _applied.erase(rightt); - _routeCmd("del",rightt,_via,(_via) ? (const char *)0 : _device); - } - if (!_applied.count(_target)) { - _applied.insert(_target); - _routeCmd("replace",_target,_via,(_via) ? (const char *)0 : _device); - } - }*/ + if (!_applied.count(leftt)) { + _applied[leftt] = false; // boolean unused + _routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device); + } + if ((rightt)&&(!_applied.count(rightt))) { + _applied[rightt] = false; // boolean unused + _routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device); + } #endif // __LINUX__ ---------------------------------------------------------- #ifdef __WINDOWS__ // -------------------------------------------------------- - //if (needBifurcation) { - if (!_applied.count(leftt)) { - _applied.insert(leftt); - _winRoute(false,interfaceLuid,interfaceIndex,leftt,_via); - } - if ((rightt)&&(!_applied.count(rightt))) { - _applied.insert(rightt); - _winRoute(false,interfaceLuid,interfaceIndex,rightt,_via); - } - /*if (_applied.count(_target)) { - _applied.erase(_target); - _winRoute(true,interfaceLuid,interfaceIndex,_target,_via); - }*/ - /*} else { - if (_applied.count(leftt)) { - _applied.erase(leftt); - _winRoute(true,interfaceLuid,interfaceIndex,leftt,_via); - } - if ((rightt)&&(_applied.count(rightt))) { - _applied.erase(rightt); - _winRoute(true,interfaceLuid,interfaceIndex,rightt,_via); - } - if (!_applied.count(_target)) { - _applied.insert(_target); - _winRoute(false,interfaceLuid,interfaceIndex,_target,_via); - } - }*/ + if (!_applied.count(leftt)) { + _applied[leftt] = false; // boolean unused + _winRoute(false,interfaceLuid,interfaceIndex,leftt,_via); + } + if ((rightt)&&(!_applied.count(rightt))) { + _applied[rightt] = false; // boolean unused + _winRoute(false,interfaceLuid,interfaceIndex,rightt,_via); + } #endif // __WINDOWS__ -------------------------------------------------------- @@ -553,9 +515,9 @@ void ManagedRoute::remove() } #endif // __BSD__ ------------------------------------------------------------ - for(std::set::iterator r(_applied.begin());r!=_applied.end();++r) { + for(std::map::iterator r(_applied.begin());r!=_applied.end();++r) { #ifdef __BSD__ // ------------------------------------------------------------ - _routeCmd("delete",*r,_via,(const char *)0,(_via) ? (const char *)0 : _device); + _routeCmd("delete",r->first,_via,r->second ? _device : (const char *)0,(_via) ? (const char *)0 : _device); #endif // __BSD__ ------------------------------------------------------------ #ifdef __LINUX__ // ---------------------------------------------------------- diff --git a/osdep/ManagedRoute.hpp b/osdep/ManagedRoute.hpp index 9c7e8477..4bf56503 100644 --- a/osdep/ManagedRoute.hpp +++ b/osdep/ManagedRoute.hpp @@ -9,7 +9,7 @@ #include #include -#include +#include namespace ZeroTier { @@ -105,7 +105,7 @@ private: InetAddress _target; InetAddress _via; InetAddress _systemVia; // for route overrides - std::set _applied; // routes currently applied + std::map _applied; // routes currently applied char _device[128]; char _systemDevice[128]; // for route overrides }; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index b443a7fa..5c135636 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -215,7 +215,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) const char *prole = ""; switch(peer->role) { case ZT_PEER_ROLE_LEAF: prole = "LEAF"; break; - case ZT_PEER_ROLE_RELAY: prole = "RELAY"; break; + case ZT_PEER_ROLE_UPSTREAM: prole = "UPSTREAM"; break; case ZT_PEER_ROLE_ROOT: prole = "ROOT"; break; } -- cgit v1.2.3 From 83abc00aaedfa2eb9698af496bfa2f1891401726 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 13 Sep 2016 14:58:59 -0700 Subject: docs --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 146f2962..5e3dae90 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1299,7 +1299,7 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou for(std::vector

::const_iterator a(anchors.begin());a!=anchors.end();++a) _membership(*a); - // Send MULTICAST_LIKE(s) to all members of this network + // Send credentials and multicast LIKEs to members, upstreams, and controller { Address *a = (Address *)0; Membership *m = (Membership *)0; -- cgit v1.2.3 From 15402933bc93f930f9767f6b2fb16fdb29c3c50e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 14 Sep 2016 16:55:25 -0700 Subject: Add physical MTU recommendation hint to network config via API. --- include/ZeroTierOne.h | 8 ++++++++ node/Network.cpp | 1 + 2 files changed, 9 insertions(+) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e4ea92b4..e0f6ca28 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -977,6 +977,11 @@ typedef struct */ unsigned int mtu; + /** + * Recommended MTU to avoid fragmentation at the physical layer (hint) + */ + unsigned int physicalMtu; + /** * If nonzero, the network this port belongs to indicates DHCP availability * @@ -1604,6 +1609,9 @@ typedef int (*ZT_PathCheckFunction)( * Note that this can take a few seconds the first time it's called, as it * will generate an identity. * + * TODO: should consolidate function pointers into versioned structure for + * better API stability. + * * @param node Result: pointer is set to new node instance on success * @param uptr User pointer to pass to functions/callbacks * @param now Current clock in milliseconds diff --git a/node/Network.cpp b/node/Network.cpp index 5e3dae90..22aca0d8 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1224,6 +1224,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const ec->status = _status(); ec->type = (_config) ? (_config.isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE; ec->mtu = ZT_IF_MTU; + ec->physicalMtu = ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 16); ec->dhcp = 0; std::vector
ab(_config.activeBridges()); ec->bridge = ((_config.allowPassiveBridging())||(std::find(ab.begin(),ab.end(),RR->identity.address()) != ab.end())) ? 1 : 0; -- cgit v1.2.3 From 740b34124f6e1d1092e8d22b1c03ee2c1beef4e7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 14 Sep 2016 17:35:50 -0700 Subject: Naming... --- node/World.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/World.hpp b/node/World.hpp index 82ee0d0e..2f1edb00 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -150,7 +150,7 @@ public: if (fullSignatureCheck) { Buffer tmp; update.serialize(tmp,true); - return C25519::verify(_updateSigningKey,tmp.data(),tmp.size(),update._signature); + return C25519::verify(_updatesMustBeSignedBy,tmp.data(),tmp.size(),update._signature); } else return true; } return false; @@ -169,7 +169,7 @@ public: b.append((uint8_t)0x01); b.append((uint64_t)_id); b.append((uint64_t)_ts); - b.append(_updateSigningKey.data,ZT_C25519_PUBLIC_KEY_LEN); + b.append(_updatesMustBeSignedBy.data,ZT_C25519_PUBLIC_KEY_LEN); if (!forSign) b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); b.append((uint8_t)_roots.size()); @@ -195,7 +195,7 @@ public: _id = b.template at(p); p += 8; _ts = b.template at(p); p += 8; - memcpy(_updateSigningKey.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN; + memcpy(_updatesMustBeSignedBy.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN; memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; unsigned int numRoots = b[p++]; if (numRoots > ZT_WORLD_MAX_ROOTS) @@ -216,13 +216,13 @@ public: return (p - startAt); } - inline bool operator==(const World &w) const throw() { return ((_id == w._id)&&(_ts == w._ts)&&(_updateSigningKey == w._updateSigningKey)&&(_signature == w._signature)&&(_roots == w._roots)); } + inline bool operator==(const World &w) const throw() { return ((_id == w._id)&&(_ts == w._ts)&&(_updatesMustBeSignedBy == w._updatesMustBeSignedBy)&&(_signature == w._signature)&&(_roots == w._roots)); } inline bool operator!=(const World &w) const throw() { return (!(*this == w)); } protected: uint64_t _id; uint64_t _ts; - C25519::Public _updateSigningKey; + C25519::Public _updatesMustBeSignedBy; C25519::Signature _signature; std::vector _roots; }; -- cgit v1.2.3 From 68e549233ddba17ec686a0462e2630580ee3d2a7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 15 Sep 2016 13:17:37 -0700 Subject: Revise bearer token code in controller, and add relay policy as a meta-data item presented to controller by nodes (to facilitate future meshiness). --- controller/EmbeddedNetworkController.cpp | 103 +++++++++++++++++++------------ controller/EmbeddedNetworkController.hpp | 4 +- controller/README.md | 4 +- node/Network.cpp | 2 + node/NetworkConfig.hpp | 10 ++- 5 files changed, 77 insertions(+), 46 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 861792ed..53b345b4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -566,42 +566,69 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; - if (!_jB(network["private"],true)) { + if (_jB(member["authorized"],false)) { + authorizedBy = "memberIsAuthorized"; + } else if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; - // If member already has an authorized field, leave it alone. That way its state is - // preserved if the user toggles the network back to private. Otherwise set it to - // true by default for new members of public nets. if (!member.count("authorized")) { member["authorized"] = true; - member["lastAuthorizedTime"] = now; - member["lastAuthorizedBy"] = authorizedBy; + json ah; + ah["a"] = true; + ah["by"] = authorizedBy; + ah["ts"] = now; + ah["ct"] = json(); + ah["c"] = json(); + member["authHistory"].push_back(ah); member["lastModified"] = now; - auto revj = member["revision"]; + json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } - } else if (_jB(member["authorized"],false)) { - authorizedBy = "memberIsAuthorized"; } else { - char atok[256]; - if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { - atok[255] = (char)0; // not necessary but YDIFLO - if (strlen(atok) > 0) { // extra sanity check since we never want to compare a null token on either side - auto authTokens = network["authTokens"]; + char presentedAuth[512]; + if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) { + presentedAuth[511] = (char)0; // sanity check + + // Check for bearer token presented by member + if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) { + const char *const presentedToken = presentedAuth + 6; + + json &authTokens = network["authTokens"]; if (authTokens.is_array()) { for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { - authorizedBy = "token"; - member["authorized"] = true; // tokens actually change member authorization state - member["lastAuthorizedTime"] = now; - member["lastAuthorizedBy"] = authorizedBy; - member["lastModified"] = now; - auto revj = member["revision"]; - member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - break; + json &token = authTokens[i]; + if (token.is_object()) { + const uint64_t expires = _jI(token["expires"],0ULL); + const uint64_t maxUses = _jI(token["maxUsesPerMember"],0ULL); + std::string tstr = _jS(token["token"],""); + + if (((expires == 0ULL)||(expires > now))&&(tstr == presentedToken)) { + bool usable = (maxUses == 0); + if (!usable) { + uint64_t useCount = 0; + json &ahist = member["authHistory"]; + if (ahist.is_array()) { + for(unsigned long j=0;j 0) { json t = json::object(); t["token"] = tstr; t["expires"] = _jI(token["expires"],0ULL); + t["maxUsesPerMember"] = _jI(token["maxUsesPerMember"],0ULL); nat.push_back(t); } } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 303a5355..1bfd9577 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -143,9 +143,7 @@ private: inline void _initMember(nlohmann::json &member) { if (!member.count("authorized")) member["authorized"] = false; - if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL; - if (!member.count("lastAuthorizedBy")) member["lastAuthorizedBy"] = ""; - if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL; + if (!member.count("authHistory")) member["authHistory"] = nlohmann::json::array(); if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); if (!member.count("activeBridge")) member["activeBridge"] = false; diff --git a/controller/README.md b/controller/README.md index 1eb0ca0d..805641d9 100644 --- a/controller/README.md +++ b/controller/README.md @@ -229,9 +229,7 @@ This returns an object containing all currently online members and the most rece | nwid | string | 16-digit network ID | no | | clock | integer | Current clock, ms since epoch | no | | authorized | boolean | Is member authorized? (for private networks) | YES | -| lastAuthorizedTime | integer | Time 'authorized' was last set to 'true' | no | -| lastAuthorizedBy | string | What last set 'authorized' to 'true'? | no | -| lastDeauthorizedTime | integer | Time 'authorized' was last set to 'false' | no | +| authHistory | array[object] | History of auth changes, latest at end | no | | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | diff --git a/node/Network.cpp b/node/Network.cpp index 22aca0d8..197841d9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1001,6 +1001,7 @@ void Network::requestConfiguration() Dictionary rmd; rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR,(uint64_t)ZT_VENDOR_ZEROTIER); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MINOR); @@ -1011,6 +1012,7 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); + rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY,(uint64_t)RR->node->relayPolicy()); if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index ad1cafa5..5ad86855 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -108,9 +108,13 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v" // Protocol version (see Packet.hpp) #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv" -// Software major, minor, revision +// Software vendor +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR "vend" +// Software major version #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv" +// Software minor version #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv" +// Software revision #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv" // Rules engine revision #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV "revr" @@ -123,9 +127,11 @@ namespace ZeroTier { // Maximum number of tags this node can accept #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS "mt" // Network join authorization token (if any) -#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN "atok" +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH "a" // Network configuration meta-data flags #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS "f" +// Relay policy for this node +#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY "rp" // These dictionary keys are short so they don't take up much room. // By convention we use upper case for binary blobs, but it doesn't really matter. -- cgit v1.2.3 From d3524f36090c47e11c2647f022b03e27d16aeb13 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 20 Sep 2016 21:21:34 -0700 Subject: Refactor COM stuff a bit, and respond to COM requests a bit more readily for rapid setup. Will need to revisit later. --- node/Constants.hpp | 7 +++- node/IncomingPacket.cpp | 25 +++++------- node/Membership.hpp | 22 +---------- node/Network.cpp | 102 ++++++++++++++++++++++++------------------------ node/Network.hpp | 11 ++---- node/Node.cpp | 2 +- node/Peer.cpp | 8 +--- node/Peer.hpp | 35 ++++++----------- node/SelfAwareness.cpp | 14 +------ node/Switch.cpp | 11 +++--- node/Topology.cpp | 11 +----- 11 files changed, 94 insertions(+), 154 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index b3c3dec0..b7042d5d 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -296,7 +296,12 @@ /** * General rate limit timeout for multiple packet types (HELLO, etc.) */ -#define ZT_PEER_GENERAL_INBOUND_RATE_LIMIT 1000 +#define ZT_PEER_GENERAL_INBOUND_RATE_LIMIT 500 + +/** + * General limit for max RTT for requests over the network + */ +#define ZT_GENERAL_RTT_LIMIT 5000 /** * Delay between requests for updated network autoconf information diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 9bc41d47..b3925773 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -153,28 +153,21 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr break; case Packet::ERROR_IDENTITY_COLLISION: - // Roots are the only peers currently permitted to state authoritatively - // that an identity has collided. When this occurs the node should be shut - // down and a new identity created. The odds of this ever happening are - // very low. + // FIXME: for federation this will need a payload with a signature or something. if (RR->topology->isRoot(peer->identity())) RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { - // This error can be sent in response to any packet that fails network - // authorization. We only listen to it if it's from a peer that has recently - // been authorized on this network. + // Peers can send this in response to frames if they do not have a recent enough COM from us SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->recentlyAllowedOnNetwork(peer))) { - const uint64_t now = RR->node->now(); - if (peer->rateGateComRequest(now)) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - network->config().com.serialize(outp); - outp.append((uint8_t)0); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),now); - } + const uint64_t now = RR->node->now(); + if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + network->config().com.serialize(outp); + outp.append((uint8_t)0); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),now); } } break; diff --git a/node/Membership.hpp b/node/Membership.hpp index d67c6822..5eb68d34 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -154,23 +154,6 @@ public: return nconf.com.agreesWith(_com); } - /** - * @return True if this member has been on this network recently (or network is public) - */ - inline bool recentlyAllowedOnNetwork(const NetworkConfig &nconf) const - { - if (nconf.isPublic()) - return true; - if (_com) { - const uint64_t a = _com.timestamp().first; - if ((_blacklistBefore)&&(a <= _blacklistBefore)) - return false; - const uint64_t b = nconf.com.timestamp().first; - return ((a <= b) ? ((b - a) <= ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) : true); - } - return false; - } - /** * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time * @@ -259,10 +242,7 @@ public: * * @param ts Blacklist cutoff */ - inline void blacklistBefore(const uint64_t ts) - { - _blacklistBefore = ts; - } + inline void blacklistBefore(const uint64_t ts) { _blacklistBefore = ts; } /** * Clean up old or stale entries diff --git a/node/Network.cpp b/node/Network.cpp index 197841d9..455e185e 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -648,11 +648,12 @@ bool Network::filterOutgoingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; - Address ztDest2(ztDest); + Address ztFinalDest(ztDest); Address cc; const Capability *relevantCap = (const Capability *)0; unsigned int ccLength = 0; bool accept = false; + const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); @@ -663,26 +664,27 @@ bool Network::filterOutgoingPacket( remoteTagCount = m->getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); } - switch(_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + switch(_doZtFilter(RR,_config,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { case DOZTFILTER_NO_MATCH: for(unsigned int c=0;c<_config.capabilityCount;++c) { - ztDest2 = ztDest; // sanity check + ztFinalDest = ztDest; // sanity check Address cc2; unsigned int ccLength2 = 0; - switch (_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + switch (_doZtFilter(RR,_config,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; - case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side relevantCap = &(_config.capabilities[c]); accept = true; if ((!noTee)&&(cc2)) { - _membership(cc2).sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,relevantCap); + Membership &m2 = _membership(cc2); + m2.sendCredentialsIfNeeded(RR,now,cc2,_config,relevantCap); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -705,7 +707,7 @@ bool Network::filterOutgoingPacket( case DOZTFILTER_DROP: return false; - case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side accept = true; @@ -714,7 +716,8 @@ bool Network::filterOutgoingPacket( if (accept) { if ((!noTee)&&(cc)) { - _membership(cc).sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,relevantCap); + Membership &m2 = _membership(cc); + m2.sendCredentialsIfNeeded(RR,now,cc,_config,relevantCap); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -727,10 +730,11 @@ bool Network::filterOutgoingPacket( RR->sw->send(outp,true); } - if ((ztDest != ztDest2)&&(ztDest2)) { - _membership(ztDest2).sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,relevantCap); + if ((ztDest != ztFinalDest)&&(ztFinalDest)) { + Membership &m2 = _membership(ztFinalDest); + m2.sendCredentialsIfNeeded(RR,now,ztFinalDest,_config,relevantCap); - Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); + Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 macDest.appendTo(outp); @@ -742,11 +746,13 @@ bool Network::filterOutgoingPacket( return false; // DROP locally, since we redirected } else if (m) { - m->sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,relevantCap); + m->sendCredentialsIfNeeded(RR,now,ztDest,_config,relevantCap); } - } - return accept; + return true; + } else { + return false; + } } int Network::filterIncomingPacket( @@ -761,7 +767,7 @@ int Network::filterIncomingPacket( { uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; - Address ztDest2(ztDest); + Address ztFinalDest(ztDest); Address cc; unsigned int ccLength = 0; int accept = 0; @@ -771,16 +777,16 @@ int Network::filterIncomingPacket( Membership &m = _membership(sourcePeer->address()); const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { case DOZTFILTER_NO_MATCH: { Membership::CapabilityIterator mci(m); const Capability *c; while ((c = mci.next(_config))) { - ztDest2 = ztDest; // sanity check + ztFinalDest = ztDest; // sanity check Address cc2; unsigned int ccLength2 = 0; - switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -815,7 +821,7 @@ int Network::filterIncomingPacket( case DOZTFILTER_DROP: return 0; // DROP - case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztDest2 will have been changed in _doZtFilter() + case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: accept = 1; // ACCEPT break; @@ -839,10 +845,10 @@ int Network::filterIncomingPacket( RR->sw->send(outp,true); } - if ((ztDest != ztDest2)&&(ztDest2)) { - _membership(ztDest2).sendCredentialsIfNeeded(RR,RR->node->now(),ztDest2,_config,(const Capability *)0); + if ((ztDest != ztFinalDest)&&(ztFinalDest)) { + _membership(ztFinalDest).sendCredentialsIfNeeded(RR,RR->node->now(),ztFinalDest,_config,(const Capability *)0); - Packet outp(ztDest2,RR->identity.address(),Packet::VERB_EXT_FRAME); + Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 macDest.appendTo(outp); @@ -1063,23 +1069,26 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin Mutex::Lock _l(_lock); try { if (_config) { - Membership &m = _membership(peer->address()); - const bool allow = m.isAllowedOnNetwork(_config); - if (allow) { - m.sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); - if (m.shouldLikeMulticasts(now)) { + Membership *m = _memberships.get(peer->address()); + if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { + if (!m) + m = &(_membership(peer->address())); + m->sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); + if (m->shouldLikeMulticasts(now)) { _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); - m.likingMulticasts(now); + m->likingMulticasts(now); + } + return true; + } else { + if (peer->rateGateRequestCredentials(now)) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((uint8_t)verb); + outp.append(packetId); + outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); + outp.append(_id); + RR->sw->send(outp,true); } - } else if (m.recentlyAllowedOnNetwork(_config)&&peer->rateGateRequestCredentials(now)) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((uint8_t)verb); - outp.append(packetId); - outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); - outp.append(_id); - RR->sw->send(outp,true); } - return allow; } } catch ( ... ) { TRACE("gate() check failed for peer %s: unexpected exception",peer->address().toString().c_str()); @@ -1092,15 +1101,6 @@ bool Network::gateMulticastGatherReply(const SharedPtr &peer,const Packet: return ( (peer->address() == controller()) || RR->topology->isUpstream(peer->identity()) || gate(peer,verb,packetId) || _config.isAnchor(peer->address()) ); } -bool Network::recentlyAllowedOnNetwork(const SharedPtr &peer) const -{ - Mutex::Lock _l(_lock); - const Membership *m = _memberships.get(peer->address()); - if (m) - return m->recentlyAllowedOnNetwork(_config); - return false; -} - void Network::clean() { const uint64_t now = RR->node->now(); @@ -1308,13 +1308,11 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - if ( (m->recentlyAllowedOnNetwork(_config)) || (std::find(anchors.begin(),anchors.end(),*a) != anchors.end()) ) { - m->sendCredentialsIfNeeded(RR,RR->node->now(),*a,_config,(const Capability *)0); - if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { - if (!newMulticastGroup) - m->likingMulticasts(now); - _announceMulticastGroupsTo(*a,groups); - } + m->sendCredentialsIfNeeded(RR,now,*a,_config,(const Capability *)0); + if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { + if (!newMulticastGroup) + m->likingMulticasts(now); + _announceMulticastGroupsTo(*a,groups); } } } diff --git a/node/Network.hpp b/node/Network.hpp index 7a4065ff..c85e5993 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -248,7 +248,10 @@ public: void requestConfiguration(); /** - * Membership check gate for incoming packets related to this network + * Determine whether this peer is permitted to communicate on this network + * + * This also performs certain periodic actions such as pushing renewed + * credentials to peers or requesting them if not present. * * @param peer Peer to check * @param verb Packet verb @@ -262,12 +265,6 @@ public: */ bool gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); - /** - * @param peer Peer to check - * @return True if peer has recently been a valid member of this network - */ - bool recentlyAllowedOnNetwork(const SharedPtr &peer) const; - /** * Perform cleanup and possibly save state */ diff --git a/node/Node.cpp b/node/Node.cpp index 51f1b5c0..2533eeb6 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -241,7 +241,7 @@ public: } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); - } else if (p->activelyTransferringFrames(_now)) { + } else if (p->isActive(_now)) { // Normal nodes get their preferred link kept alive if the node has generated frame traffic recently p->doPingAndKeepalive(_now,-1); } diff --git a/node/Peer.cpp b/node/Peer.cpp index 78af9063..d742964a 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -41,7 +41,6 @@ namespace ZeroTier { static uint32_t _natKeepaliveBuf = 0; Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : - _lastUsed(0), _lastReceive(0), _lastUnicastFrame(0), _lastMulticastFrame(0), @@ -408,19 +407,16 @@ bool Peer::hasActiveDirectPath(uint64_t now) const return false; } -bool Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) +void Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) { Mutex::Lock _l(_paths_m); - bool resetSomething = false; for(unsigned int p=0;p<_numPaths;++p) { if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now); _paths[p].path->sent(now); - _paths[p].lastReceive >>= 2; // de-prioritize heavily vs. other paths, will get reset if we get OK(HELLO) or other traffic - resetSomething = true; + _paths[p].lastReceive = 0; // path will not be used unless it speaks again } } - return resetSomething; } void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const diff --git a/node/Peer.hpp b/node/Peer.hpp index 1ae239bc..c5ef43ed 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -68,18 +68,6 @@ public: */ Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity); - /** - * @return Time peer record was last used in any way - */ - inline uint64_t lastUsed() const throw() { return _lastUsed; } - - /** - * Log a use of this peer record (done by Topology when peers are looked up) - * - * @param now New time of last use - */ - inline void use(uint64_t now) throw() { _lastUsed = now; } - /** * @return This peer's ZT address (short for identity().address()) */ @@ -194,15 +182,14 @@ public: /** * Reset paths within a given IP scope and address family * - * Resetting a path involves sending a HELLO to it and then de-prioritizing - * it vs. other paths. + * Resetting a path involves sending an ECHO to it and then deactivating + * it until or unless it responds. * * @param scope IP scope * @param inetAddressFamily Family e.g. AF_INET * @param now Current time - * @return True if we forgot at least one path */ - bool resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); + void resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); /** * Get most recently active path addresses for IPv4 and/or IPv6 @@ -232,27 +219,32 @@ public: /** * @return Time of last receive of anything, whether direct or relayed */ - inline uint64_t lastReceive() const throw() { return _lastReceive; } + inline uint64_t lastReceive() const { return _lastReceive; } + + /** + * @return True if we've heard from this peer in less than ZT_PEER_ACTIVITY_TIMEOUT + */ + inline bool isAlive(const uint64_t now) const { return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return Time of most recent unicast frame received */ - inline uint64_t lastUnicastFrame() const throw() { return _lastUnicastFrame; } + inline uint64_t lastUnicastFrame() const { return _lastUnicastFrame; } /** * @return Time of most recent multicast frame received */ - inline uint64_t lastMulticastFrame() const throw() { return _lastMulticastFrame; } + inline uint64_t lastMulticastFrame() const { return _lastMulticastFrame; } /** * @return Time of most recent frame of any kind (unicast or multicast) */ - inline uint64_t lastFrame() const throw() { return std::max(_lastUnicastFrame,_lastMulticastFrame); } + inline uint64_t lastFrame() const { return std::max(_lastUnicastFrame,_lastMulticastFrame); } /** * @return True if this peer has sent us real network traffic recently */ - inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline uint64_t isActive(uint64_t now) const { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return Latency in milliseconds or 0 if unknown @@ -464,7 +456,6 @@ private: uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; uint8_t _remoteClusterOptimal6[16]; - uint64_t _lastUsed; uint64_t _lastReceive; // direct or indirect uint64_t _lastUnicastFrame; uint64_t _lastMulticastFrame; diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 6bf50720..e84b7b65 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -45,9 +45,7 @@ public: _family(inetAddressFamily), _scope(scope) {} - inline void operator()(Topology &t,const SharedPtr &p) { if (p->resetWithinScope(_scope,_family,_now)) peersReset.push_back(p); } - - std::vector< SharedPtr > peersReset; + inline void operator()(Topology &t,const SharedPtr &p) { p->resetWithinScope(_scope,_family,_now); } private: uint64_t _now; @@ -95,16 +93,6 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc // Reset all paths within this scope and address family _ResetWithinScope rset(now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope); RR->topology->eachPeer<_ResetWithinScope &>(rset); - - // Send a NOP to all peers for whom we forgot a path. This will cause direct - // links to be re-established if possible, possibly using a root server or some - // other relay. - for(std::vector< SharedPtr >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) { - if ((*p)->activelyTransferringFrames(now)) { - Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP); - RR->sw->send(outp,true); - } - } } else { // Otherwise just update DB to use to determine external surface info entry.mySurface = myPhysicalAddress; diff --git a/node/Switch.cpp b/node/Switch.cpp index beb36b6c..e3d57835 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -354,8 +354,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c return; } - // Destination is a multicast address (including broadcast) - MulticastGroup mg(to,0); + MulticastGroup multicastGroup(to,0); if (to.isBroadcast()) { if ( (etherType == ZT_ETHERTYPE_ARP) && (len >= 28) && ((((const uint8_t *)data)[2] == 0x08)&&(((const uint8_t *)data)[3] == 0x00)&&(((const uint8_t *)data)[4] == 6)&&(((const uint8_t *)data)[5] == 4)&&(((const uint8_t *)data)[7] == 0x01)) ) { @@ -368,7 +367,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c * them into multicasts by stuffing the IP address being queried into * the 32-bit ADI field. In practice this uses our multicast pub/sub * system to implement a kind of extended/distributed ARP table. */ - mg = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(((const unsigned char *)data) + 24,4,0)); + multicastGroup = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(((const unsigned char *)data) + 24,4,0)); } else if (!network->config().enableBroadcast()) { // Don't transmit broadcasts if this network doesn't want them TRACE("%.16llx: dropped broadcast since ff:ff:ff:ff:ff:ff is not enabled",network->id()); @@ -463,9 +462,9 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c * multicast addresses on bridge interfaces and subscribing each slave. * But in that case this does no harm, as the sets are just merged. */ if (fromBridged) - network->learnBridgedMulticastGroup(mg,RR->node->now()); + network->learnBridgedMulticastGroup(multicastGroup,RR->node->now()); - //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),len); + //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),multicastGroup.toString().c_str(),etherTypeName(etherType),len); // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { @@ -478,7 +477,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c RR->node->now(), network->id(), network->config().activeBridges(), - mg, + multicastGroup, (fromBridged) ? from : MAC(), etherType, data, diff --git a/node/Topology.cpp b/node/Topology.cpp index 6e2fd071..12a7cc0b 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -96,7 +96,6 @@ SharedPtr Topology::addPeer(const SharedPtr &peer) np = hp; } - np->use(RR->node->now()); saveIdentity(np->identity()); return np; @@ -113,7 +112,6 @@ SharedPtr Topology::getPeer(const Address &zta) Mutex::Lock _l(_lock); const SharedPtr *const ap = _peers.get(zta); if (ap) { - (*ap)->use(RR->node->now()); return *ap; } } @@ -127,7 +125,6 @@ SharedPtr Topology::getPeer(const Address &zta) SharedPtr &ap = _peers[zta]; if (!ap) ap.swap(np); - ap->use(RR->node->now()); return ap; } } @@ -176,10 +173,8 @@ SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCou if (_rootAddresses[p] == RR->identity.address()) { for(unsigned long q=1;q<_rootAddresses.size();++q) { const SharedPtr *const nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]); - if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) { - (*nextsn)->use(now); + if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) return *nextsn; - } } break; } @@ -214,10 +209,8 @@ SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCou } if (bestNotAvoid) { - (*bestNotAvoid)->use(now); return *bestNotAvoid; } else if ((!strictAvoid)&&(bestOverall)) { - (*bestOverall)->use(now); return *bestOverall; } @@ -256,7 +249,7 @@ void Topology::clean(uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) + if ( (!(*p)->isAlive(now)) && (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) ) _peers.erase(*a); } } -- cgit v1.2.3 From 1f74dd4589017bd5bc34b31ff6e09e6875d5e1c7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 23 Sep 2016 16:08:38 -0700 Subject: Revocation work in progress, add WATCH which is TEE with implicit rate sync (thanks JG@DCVC!), and clean up some cruft in Network. --- controller/EmbeddedNetworkController.cpp | 12 ++ include/ZeroTierOne.h | 9 +- node/Capability.hpp | 2 + node/IncomingPacket.cpp | 76 ++++---- node/Membership.cpp | 234 +++++++++++++++---------- node/Membership.hpp | 263 +++++++++++++--------------- node/Network.cpp | 289 +++++++++++++------------------ node/Network.hpp | 189 +++++++------------- node/Packet.hpp | 63 ++++--- node/Revocation.cpp | 46 +++++ node/Revocation.hpp | 178 +++++++++++++++++++ node/Tag.cpp | 2 +- node/Tag.hpp | 30 +++- objects.mk | 1 + 14 files changed, 803 insertions(+), 591 deletions(-) create mode 100644 node/Revocation.cpp create mode 100644 node/Revocation.hpp (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 53b345b4..5ba8cf98 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -140,6 +140,12 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["flags"] = (unsigned int)rule.v.fwd.flags; r["length"] = (unsigned int)rule.v.fwd.length; break; + case ZT_NETWORK_RULE_ACTION_WATCH: + r["type"] = "ACTION_WATCH"; + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (unsigned int)rule.v.fwd.flags; + r["length"] = (unsigned int)rule.v.fwd.length; + break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; r["address"] = Address(rule.v.fwd.address).toString(); @@ -303,6 +309,12 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; + } else if (t == "ACTION_WATCH") { + rule.t |= ZT_NETWORK_RULE_ACTION_WATCH; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); + return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e0f6ca28..e43c8541 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -516,15 +516,20 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_ACTION_TEE = 2, + /** + * Exactly like TEE but frames are dropped if previous TEEs were not acknowledged by the observer + */ + ZT_NETWORK_RULE_ACTION_WATCH = 3, + /** * Drop and redirect this frame to another node (by ZT address) */ - ZT_NETWORK_RULE_ACTION_REDIRECT = 3, + ZT_NETWORK_RULE_ACTION_REDIRECT = 4, /** * Log if match and if rule debugging is enabled in the build, otherwise does nothing (for developers) */ - ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 4, + ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 5, /** * Maximum ID for an ACTION, anything higher is a MATCH diff --git a/node/Capability.hpp b/node/Capability.hpp index e23d7943..2cf54b5c 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -174,6 +174,7 @@ public: b.append((uint8_t)0); break; case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: case ZT_NETWORK_RULE_ACTION_REDIRECT: b.append((uint8_t)14); b.append((uint64_t)rules[i].v.fwd.address); @@ -270,6 +271,7 @@ public: default: break; case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: case ZT_NETWORK_RULE_ACTION_REDIRECT: rules[ruleCount].v.fwd.address = b.template at(p); rules[ruleCount].v.fwd.flags = b.template at(p + 8); diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b3925773..12766fe2 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -39,6 +39,7 @@ #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" +#include "Revocation.hpp" namespace ZeroTier { @@ -162,13 +163,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // Peers can send this in response to frames if they do not have a recent enough COM from us SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); const uint64_t now = RR->node->now(); - if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - network->config().com.serialize(outp); - outp.append((uint8_t)0); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),now); - } + if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) + network->pushCredentialsNow(peer->address(),now); } break; case Packet::ERROR_NETWORK_ACCESS_DENIED_: { @@ -681,9 +677,17 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

node->putFrame(nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; } + } - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + if ((flags & 0x10) != 0) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((uint8_t)Packet::VERB_EXT_FRAME); + outp.append((uint64_t)packetId()); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); } + + peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); @@ -775,6 +779,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S CertificateOfMembership com; Capability cap; Tag tag; + Revocation revocation; bool trustEstablished = false; unsigned int p = ZT_PACKET_IDX_PAYLOAD; @@ -784,8 +789,14 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S SharedPtr network(RR->node->network(com.networkId())); if (network) { switch (network->addCredential(com)) { - case 0: trustEstablished = true; break; - case 1: return false; // wait for WHOIS + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } else RR->mc->addCredential(com,false); } @@ -799,8 +810,14 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S SharedPtr network(RR->node->network(cap.networkId())); if (network) { switch (network->addCredential(cap)) { - case 0: trustEstablished = true; break; - case 1: return false; // wait for WHOIS + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } } @@ -811,11 +828,25 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S SharedPtr network(RR->node->network(tag.networkId())); if (network) { switch (network->addCredential(tag)) { - case 0: trustEstablished = true; break; - case 1: return false; // wait for WHOIS + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } } + + const unsigned int numRevocations = at(p); p += 2; + for(unsigned int i=0;i network(RR->node->network(revocation.networkId())); + if (network) { + } + } } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); @@ -932,24 +963,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons const uint64_t nwid = at(ZT_PACKET_IDX_PAYLOAD); bool trustEstablished = false; - if (Network::controllerFor(nwid) == peer->address()) { - SharedPtr network(RR->node->network(nwid)); - if (network) { - network->requestConfiguration(); - trustEstablished = true; - } else { - TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false); - return true; - } - const unsigned int blacklistCount = at(ZT_PACKET_IDX_PAYLOAD + 8); - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 10; - for(unsigned int i=0;iblacklistBefore(Address(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH),at(ptr + 5)); - ptr += 13; - } - } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { diff --git a/node/Membership.cpp b/node/Membership.cpp index 8c2ba673..d579d303 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -16,6 +16,8 @@ * along with this program. If not, see . */ +#include + #include "Membership.hpp" #include "RuntimeEnvironment.hpp" #include "Peer.hpp" @@ -28,28 +30,43 @@ namespace ZeroTier { -void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap) +Membership::Membership() : + _lastUpdatedMulticast(0), + _lastPushAttempt(0), + _lastPushedCom(0), + _comRevocationThreshold(0) +{ + for(unsigned int i=0;i= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); do { - unfinished = false; Buffer capsAndTags; unsigned int appendedCaps = 0; - if (cap) { + if (localCapabilityIndex >= 0) { capsAndTags.addSize(2); - std::map::iterator cs(_caps.find(cap->id())); - if ((cs != _caps.end())&&((now - cs->second.lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY)) { - cap->serialize(capsAndTags); - cs->second.lastPushed = now; + + if ( (_localCaps[localCapabilityIndex].id != nconf.capabilities[localCapabilityIndex].id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCaps[localCapabilityIndex].lastPushed = now; + _localCaps[localCapabilityIndex].id = nconf.capabilities[localCapabilityIndex].id(); + nconf.capabilities[localCapabilityIndex].serialize(capsAndTags); ++appendedCaps; } + capsAndTags.setAt(0,(uint16_t)appendedCaps); + localCapabilityIndex = -1; // don't send this cap again on subsequent loops if force is true } else { capsAndTags.append((uint16_t)0); } @@ -57,22 +74,17 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint unsigned int appendedTags = 0; const unsigned int tagCountPos = capsAndTags.size(); capsAndTags.addSize(2); - for(unsigned int i=0;ilastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) { - if ((capsAndTags.size() + sizeof(Tag)) >= (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) { - unfinished = true; + for(;localTagPtr= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + if ((capsAndTags.size() + sizeof(Tag)) >= (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) break; - } - nconf.tags[i].serialize(capsAndTags); - ts->lastPushed = now; + nconf.tags[localTagPtr].serialize(capsAndTags); ++appendedTags; } } capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - const bool needCom = ((nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY)); - if ( (needCom) || (appendedCaps) || (appendedTags) ) { + if (needCom||appendedCaps||appendedTags) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (needCom) { nconf.com.serialize(outp); @@ -80,110 +92,148 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint } outp.append((uint8_t)0x00); outp.append(capsAndTags.data(),capsAndTags.size()); + outp.append((uint16_t)0); // no revocations, these propagate differently outp.compress(); RR->sw->send(outp,true); + needCom = false; // don't send COM again on subsequent loops if force is true } - } while (unfinished); // if there are many tags, etc., we can send more than one + } while (localTagPtr < nconf.tagCount); } catch ( ... ) { TRACE("unable to send credentials due to unexpected exception"); } } -int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com) +const Capability *Membership::getCapability(const NetworkConfig &nconf,const uint32_t id) const { - if (_com == com) { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); - return 0; + const _RemoteCapability *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialSorter<_RemoteCapability>()); + return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->cap,**c))) ? &((*c)->cap) : (const Capability *)0) : (const Capability *)0); +} + +const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) const +{ + const _RemoteTag *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialSorter<_RemoteTag>()); + return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->tag,**t))) ? &((*t)->tag) : (const Tag *)0) : (const Tag *)0); +} + +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) +{ + const uint64_t newts = com.timestamp().first; + if (newts <= _comRevocationThreshold) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (revoked)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_REJECTED; } - const int vr = com.verify(RR); + const uint64_t oldts = _com.timestamp().first; + if (newts < oldts) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (older than current)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_REJECTED; + } + if ((newts == oldts)&&(_com == com)) { + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } - if (vr == 0) { - if (com.timestamp().first >= _com.timestamp().first) { + switch(com.verify(RR)) { + default: + TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (invalid signature or object)",com.issuedTo().toString().c_str(),com.networkId()); + return ADD_REJECTED; + case 0: TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); _com = com; - } else { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED but not used (OK but older than current)",com.issuedTo().toString().c_str(),com.networkId()); - } - } else { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr); + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; } - - return vr; } -int Membership::addCredential(const RuntimeEnvironment *RR,const Tag &tag) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag) { - TState *t = _tags.get(tag.id()); - if ((t)&&(t->lastReceived != 0)&&(t->tag == tag)) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); - return 0; + _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialSorter<_RemoteTag>()); + _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteTag *)0; + if (have) { + if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->tag.timestamp() > tag.timestamp()) ) { + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + } + if (have->tag == tag) { + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } } - const int vr = tag.verify(RR); - if (vr == 0) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (!t) { - while (_tags.size() >= ZT_MAX_NETWORK_TAGS) { - uint32_t oldest = 0; - uint64_t oldestLastReceived = 0xffffffffffffffffULL; - uint32_t *i = (uint32_t *)0; - TState *ts = (TState *)0; - Hashtable::Iterator tsi(_tags); - while (tsi.next(i,ts)) { - if (ts->lastReceived < oldestLastReceived) { - oldestLastReceived = ts->lastReceived; - oldest = *i; + + switch(tag.verify(RR)) { + default: + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + case 0: + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); + if (have) { + have->lastReceived = RR->node->now(); + have->tag = tag; + } else { + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == 0xffffffffffffffffULL) { + have = _remoteTags[i]; + break; + } else if (_remoteTags[i]->lastReceived <= minlr) { + have = _remoteTags[i]; + minlr = _remoteTags[i]->lastReceived; } } - if (oldestLastReceived != 0xffffffffffffffffULL) - _tags.erase(oldest); + have->lastReceived = RR->node->now(); + have->tag = tag; + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); } - t = &(_tags[tag.id()]); - } - if (t->tag.timestamp() <= tag.timestamp()) { - t->lastReceived = RR->node->now(); - t->tag = tag; - } - } else { - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (%d)",tag.issuedTo().toString().c_str(),tag.networkId(),vr); + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; } - return vr; } -int Membership::addCredential(const RuntimeEnvironment *RR,const Capability &cap) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap) { - std::map::iterator c(_caps.find(cap.id())); - if ((c != _caps.end())&&(c->second.lastReceived != 0)&&(c->second.cap == cap)) { - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); - return 0; + _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialSorter<_RemoteCapability>()); + _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCapability *)0; + if (have) { + if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->cap.timestamp() > cap.timestamp()) ) { + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + } + if (have->cap == cap) { + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } } - const int vr = cap.verify(RR); - if (vr == 0) { - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); - if (c == _caps.end()) { - while (_caps.size() >= ZT_MAX_NETWORK_CAPABILITIES) { - std::map::iterator oldest; - uint64_t oldestLastReceived = 0xffffffffffffffffULL; - for(std::map::iterator i(_caps.begin());i!=_caps.end();++i) { - if (i->second.lastReceived < oldestLastReceived) { - oldestLastReceived = i->second.lastReceived; - oldest = i; + + switch(cap.verify(RR)) { + default: + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); + return ADD_REJECTED; + case 0: + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); + if (have) { + have->lastReceived = RR->node->now(); + have->cap = cap; + } else { + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == 0xffffffffffffffffULL) { + have = _remoteCaps[i]; + break; + } else if (_remoteCaps[i]->lastReceived <= minlr) { + have = _remoteCaps[i]; + minlr = _remoteCaps[i]->lastReceived; } } - if (oldestLastReceived != 0xffffffffffffffffULL) - _caps.erase(oldest); + have->lastReceived = RR->node->now(); + have->cap = cap; + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); } - CState &c2 = _caps[cap.id()]; - c2.lastReceived = RR->node->now(); - c2.cap = cap; - } else if (c->second.cap.timestamp() <= cap.timestamp()) { - c->second.lastReceived = RR->node->now(); - c->second.cap = cap; - } - } else { - TRACE("addCredential(Capability) for %s on %.16llx REJECTED (%d)",cap.issuedTo().toString().c_str(),cap.networkId(),vr); + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; } - return vr; } } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 5eb68d34..421e3ee8 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -21,14 +21,12 @@ #include -#include - #include "Constants.hpp" #include "../include/ZeroTierOne.h" #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" -#include "Hashtable.hpp" +#include "Revocation.hpp" #include "NetworkConfig.hpp" namespace ZeroTier { @@ -40,77 +38,135 @@ class Network; * A container for certificates of membership and other network credentials * * This is kind of analogous to a join table between Peer and Network. It is - * presently held by the Network object for each participating Peer. + * held by the Network object for each participating Peer. * - * This is not thread safe. It must be locked externally. + * This class is not thread safe. It must be locked externally. */ class Membership { private: // Tags and related state - struct TState + struct _RemoteTag { - TState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed OUR tag to this peer (with this ID) - uint64_t lastPushed; + _RemoteTag() : id(0xffffffffffffffffULL),lastReceived(0),revocationThreshold(0) {} + // Tag ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) + uint64_t id; // Last time we received THEIR tag (with this ID) uint64_t lastReceived; + // Revocation blacklist threshold or 0 if none + uint64_t revocationThreshold; // THEIR tag Tag tag; }; // Credentials and related state - struct CState + struct _RemoteCapability { - CState() : lastPushed(0),lastReceived(0) {} - // Last time we pushed OUR capability to this peer (with this ID) - uint64_t lastPushed; + _RemoteCapability() : id(0xffffffffffffffffULL),lastReceived(0),revocationThreshold(0) {} + // Capability ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) + uint64_t id; // Last time we received THEIR capability (with this ID) uint64_t lastReceived; + // Revocation blacklist threshold or 0 if none + uint64_t revocationThreshold; // THEIR capability Capability cap; }; + // Comparison operator for remote credential entries + template + struct _RemoteCredentialSorter + { + inline bool operator()(const T *a,const T *b) const { return (a->id < b->id); } + inline bool operator()(const uint64_t a,const T *b) const { return (a < b->id); } + inline bool operator()(const T *a,const uint64_t b) const { return (a->id < b); } + inline bool operator()(const uint64_t a,const uint64_t b) const { return (a < b); } + }; + + // Used to track push state for network config tags[] and capabilities[] entries + struct _LocalCredentialPushState + { + _LocalCredentialPushState() : lastPushed(0),id(0) {} + uint64_t lastPushed; + uint32_t id; + }; + public: + enum AddCredentialResult + { + ADD_REJECTED, + ADD_ACCEPTED_NEW, + ADD_ACCEPTED_REDUNDANT, + ADD_DEFERRED_FOR_WHOIS + }; + /** - * A wrapper to iterate through member capabilities in ascending order of capability ID and return only valid ones + * Iterator to scan forward through capabilities in ascending order of ID */ class CapabilityIterator { public: - CapabilityIterator(const Membership &m) : - _m(m), - _i(m._caps.begin()), - _e(m._caps.end()) - { - } + CapabilityIterator(const Membership &m,const NetworkConfig &nconf) : + _m(&m), + _c(&nconf), + _i(&(m._remoteCaps[0])) {} - inline const Capability *next(const NetworkConfig &nconf) + inline const Capability *next() { - while (_i != _e) { - if ((_i->second.lastReceived)&&(_m.isCredentialTimestampValid(nconf,_i->second.cap))) - return &((_i++)->second.cap); - else ++_i; + for(;;) { + if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != 0xffffffffffffffffULL)) { + const Capability *tmp = &((*_i)->cap); + if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { + ++_i; + return tmp; + } else ++_i; + } else { + return (const Capability *)0; + } } - return (const Capability *)0; } private: - const Membership &_m; - std::map::const_iterator _i,_e; + const Membership *_m; + const NetworkConfig *_c; + const _RemoteCapability *const *_i; }; friend class CapabilityIterator; - Membership() : - _lastUpdatedMulticast(0), - _lastPushAttempt(0), - _lastPushedCom(0), - _blacklistBefore(0), - _com(), - _caps(), - _tags(8) + /** + * Iterator to scan forward through tags in ascending order of ID + */ + class TagIterator { - } + public: + TagIterator(const Membership &m,const NetworkConfig &nconf) : + _m(&m), + _c(&nconf), + _i(&(m._remoteTags[0])) {} + + inline const Tag *next() + { + for(;;) { + if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != 0xffffffffffffffffULL)) { + const Tag *tmp = &((*_i)->tag); + if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { + ++_i; + return tmp; + } else ++_i; + } else { + return (const Tag *)0; + } + } + } + + private: + const Membership *_m; + const NetworkConfig *_c; + const _RemoteTag *const *_i; + }; + friend class TagIterator; + + Membership(); /** * Send COM and other credentials to this peer if needed @@ -122,9 +178,10 @@ public: * @param now Current time * @param peerAddress Address of member peer (the one that this Membership describes) * @param nconf My network config - * @param cap Capability to send or 0 if none + * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none + * @param force If true, send objects regardless of last push time */ - void sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap); + void pushCredentials(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** * Check whether we should push MULTICAST_LIKEs to this peer @@ -142,6 +199,8 @@ public: inline void likingMulticasts(const uint64_t now) { _lastUpdatedMulticast = now; } /** + * Check whether the peer represented by this Membership should be allowed on this network at all + * * @param nconf Our network config * @return True if this peer is allowed on this network at all */ @@ -149,126 +208,48 @@ public: { if (nconf.isPublic()) return true; - if ((_blacklistBefore)&&(_com.timestamp().first <= _blacklistBefore)) + if ((_comRevocationThreshold)&&(_com.timestamp().first <= _comRevocationThreshold)) return false; return nconf.com.agreesWith(_com); } /** - * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time - * - * @param cred Credential to check -- must have timestamp() accessor method - * @return True if credential is NOT expired + * @param nconf Network configuration + * @param id Capablity ID + * @return Pointer to capability or NULL if not found */ - template - inline bool isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred) const - { - const uint64_t ts = cred.timestamp(); - const uint64_t delta = (ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts); - return ((delta <= nconf.credentialTimeMaxDelta)&&(ts > _blacklistBefore)); - } + const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const; /** * @param nconf Network configuration * @param id Tag ID * @return Pointer to tag or NULL if not found */ - inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const - { - const TState *t = _tags.get(id); - return ((t) ? (((t->lastReceived != 0)&&(isCredentialTimestampValid(nconf,t->tag))) ? &(t->tag) : (const Tag *)0) : (const Tag *)0); - } - - /** - * @param nconf Network configuration - * @param ids Array to store IDs into - * @param values Array to store values into - * @param maxTags Capacity of ids[] and values[] - * @return Number of tags added to arrays - */ - inline unsigned int getAllTags(const NetworkConfig &nconf,uint32_t *ids,uint32_t *values,unsigned int maxTags) const - { - unsigned int n = 0; - uint32_t *id = (uint32_t *)0; - TState *ts = (TState *)0; - Hashtable::Iterator i(const_cast(this)->_tags); - while (i.next(id,ts)) { - if ((ts->lastReceived)&&(isCredentialTimestampValid(nconf,ts->tag))) { - if (n >= maxTags) - return n; - ids[n] = *id; - values[n] = ts->tag.value(); - } - } - return n; - } - - /** - * @param nconf Network configuration - * @param id Capablity ID - * @return Pointer to capability or NULL if not found - */ - inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const - { - std::map::const_iterator c(_caps.find(id)); - return ((c != _caps.end()) ? (((c->second.lastReceived != 0)&&(isCredentialTimestampValid(nconf,c->second.cap))) ? &(c->second.cap) : (const Capability *)0) : (const Capability *)0); - } + const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const; /** * Validate and add a credential if signature is okay and it's otherwise good - * - * @param RR Runtime environment - * @param com Certificate of membership - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const CertificateOfMembership &com); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good - * - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const Tag &tag); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag); /** * Validate and add a credential if signature is okay and it's otherwise good - * - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int addCredential(const RuntimeEnvironment *RR,const Capability &cap); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap); - /** - * Blacklist COM, tags, and capabilities before this time - * - * @param ts Blacklist cutoff - */ - inline void blacklistBefore(const uint64_t ts) { _blacklistBefore = ts; } - - /** - * Clean up old or stale entries - * - * @param nconf Network config - */ - inline void clean(const NetworkConfig &nconf) +private: + template + inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred,const CS &state) const { - for(std::map::iterator i(_caps.begin());i!=_caps.end();) { - if (!isCredentialTimestampValid(nconf,i->second.cap)) { - _caps.erase(i++); - } else { - ++i; - } - } - - uint32_t *i = (uint32_t *)0; - TState *ts = (TState *)0; - Hashtable::Iterator tsi(_tags); - while (tsi.next(i,ts)) { - if (!isCredentialTimestampValid(nconf,ts->tag)) - _tags.erase(*i); - } + const uint64_t ts = cred.timestamp(); + return ( (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) && (ts > state.revocationThreshold) ); } -private: // Last time we pushed MULTICAST_LIKE(s) uint64_t _lastUpdatedMulticast; @@ -278,17 +259,23 @@ private: // Last time we pushed our COM to this peer uint64_t _lastPushedCom; - // Time before which to blacklist credentials from this peer - uint64_t _blacklistBefore; + // Revocation threshold for COM or 0 if none + uint64_t _comRevocationThreshold; - // COM from this peer + // Remote member's latest network COM CertificateOfMembership _com; - // Capability-related state (we need an ordered container here, hence std::map) - std::map _caps; + // Sorted (in ascending order of ID) arrays of pointers to remote tags and capabilities + _RemoteTag *_remoteTags[ZT_MAX_NETWORK_TAGS]; + _RemoteCapability *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; + + // This is the RAM allocated for remote tags and capabilities from which the sorted arrays are populated + _RemoteTag _tagMem[ZT_MAX_NETWORK_TAGS]; + _RemoteCapability _capMem[ZT_MAX_NETWORK_CAPABILITIES]; - // Tag-related state - Hashtable _tags; + // Local credential push state tracking + _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; + _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; }; } // namespace ZeroTier diff --git a/node/Network.cpp b/node/Network.cpp index 455e185e..0fab6a27 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -36,7 +36,7 @@ #include "Peer.hpp" // Uncomment to make the rules engine dump trace info to stdout -//#define ZT_RULES_ENGINE_DEBUGGING 1 +#define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { @@ -155,24 +155,21 @@ enum _doZtFilterResult static _doZtFilterResult _doZtFilter( const RuntimeEnvironment *RR, const NetworkConfig &nconf, + const Membership *membership, // can be NULL const bool inbound, const Address &ztSource, - Address &ztDest, // MUTABLE + Address &ztDest, // MUTABLE -- is changed on REDIRECT actions const MAC &macSource, const MAC &macDest, const uint8_t *const frameData, const unsigned int frameLen, const unsigned int etherType, const unsigned int vlanId, - const ZT_VirtualNetworkRule *rules, + const ZT_VirtualNetworkRule *rules, // cannot be NULL const unsigned int ruleCount, - const Tag *localTags, - const unsigned int localTagCount, - const uint32_t *const remoteTagIds, - const uint32_t *const remoteTagValues, - const unsigned int remoteTagCount, - Address &cc, // MUTABLE - unsigned int &ccLength) // MUTABLE + Address &cc, // MUTABLE -- set to TEE destination if TEE action is taken or left alone otherwise + unsigned int &ccLength, // MUTABLE -- set to length of packet payload to TEE + bool &ccWatch) // MUTABLE -- set to true for WATCH target as opposed to normal TEE { #ifdef ZT_RULES_ENGINE_DEBUGGING char dpbuf[1024]; // used by FILTER_TRACE macro @@ -204,6 +201,7 @@ static _doZtFilterResult _doZtFilter( // These are initially handled together since preliminary logic is common case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: case ZT_NETWORK_RULE_ACTION_REDIRECT: { const Address fwdAddr(rules[rn].v.fwd.address); if (fwdAddr == ztSource) { @@ -242,6 +240,7 @@ static _doZtFilterResult _doZtFilter( #endif // ZT_RULES_ENGINE_DEBUGGING cc = fwdAddr; ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; + ccWatch = (rt == ZT_NETWORK_RULE_ACTION_WATCH); } } } continue; @@ -508,50 +507,40 @@ static _doZtFilterResult _doZtFilter( case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { - const Tag *lt = (const Tag *)0; - for(unsigned int i=0;i 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); - } else { - const uint32_t *rtv = (const uint32_t *)0; - for(unsigned int i=0;i 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); - } else { - thisRuleMatches = 1; - FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); - } - } else { + const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); + if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { + const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); + if (remoteTag) { + const uint32_t ltv = localTag->value(); + const uint32_t rtv = remoteTag->value(); if (rt == ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE) { - const uint32_t diff = (lt->value() > *rtv) ? (lt->value() - *rtv) : (*rtv - lt->value()); + const uint32_t diff = (ltv > rtv) ? (ltv - rtv) : (rtv - ltv); thisRuleMatches = (uint8_t)(diff <= rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); + FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { - thisRuleMatches = (uint8_t)((lt->value() & *rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + thisRuleMatches = (uint8_t)((ltv & rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) { - thisRuleMatches = (uint8_t)((lt->value() | *rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + thisRuleMatches = (uint8_t)((ltv | rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { - thisRuleMatches = (uint8_t)((lt->value() ^ *rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,lt->value(),*rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + thisRuleMatches = (uint8_t)((ltv ^ rtv) == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } + } else { + if (inbound) { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } else { + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } } + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } break; @@ -582,7 +571,6 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _portInitialized(false), _inboundConfigPacketId(0), _lastConfigUpdate(0), - _lastRequestedConfiguration(0), _destroyed(false), _netconfFailure(NETCONF_FAILURE_NONE), _portError(0) @@ -598,7 +586,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : if (conf.length()) { dconf->load(conf.c_str()); if (nconf->fromDictionary(*dconf)) { - this->setConfiguration(*nconf,false); + this->_setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } @@ -646,32 +634,27 @@ bool Network::filterOutgoingPacket( const unsigned int etherType, const unsigned int vlanId) { - uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; - uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; + const uint64_t now = RR->node->now(); Address ztFinalDest(ztDest); - Address cc; - const Capability *relevantCap = (const Capability *)0; - unsigned int ccLength = 0; + int localCapabilityIndex = -1; bool accept = false; - const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); - Membership *m = (Membership *)0; - unsigned int remoteTagCount = 0; - if (ztDest) { - m = &(_memberships[ztDest]); - remoteTagCount = m->getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); - } + Membership *const membership = (ztDest) ? _memberships.get(ztDest) : (Membership *)0; - switch(_doZtFilter(RR,_config,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + Address cc; + unsigned int ccLength = 0; + bool ccWatch = false; + switch(_doZtFilter(RR,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { case DOZTFILTER_NO_MATCH: for(unsigned int c=0;c<_config.capabilityCount;++c) { - ztFinalDest = ztDest; // sanity check + ztFinalDest = ztDest; // sanity check, shouldn't be possible if there was no match Address cc2; unsigned int ccLength2 = 0; - switch (_doZtFilter(RR,_config,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + bool ccWatch2 = false; + switch (_doZtFilter(RR,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -679,16 +662,16 @@ bool Network::filterOutgoingPacket( case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side - relevantCap = &(_config.capabilities[c]); + localCapabilityIndex = (int)c; accept = true; if ((!noTee)&&(cc2)) { Membership &m2 = _membership(cc2); - m2.sendCredentialsIfNeeded(RR,now,cc2,_config,relevantCap); + m2.pushCredentials(RR,now,cc2,_config,localCapabilityIndex,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + outp.append((uint8_t)(ccWatch2 ? 0x16 : 0x02)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -715,13 +698,16 @@ bool Network::filterOutgoingPacket( } if (accept) { + if (membership) + membership->pushCredentials(RR,now,ztDest,_config,localCapabilityIndex,false); + if ((!noTee)&&(cc)) { Membership &m2 = _membership(cc); - m2.sendCredentialsIfNeeded(RR,now,cc,_config,relevantCap); + m2.pushCredentials(RR,now,cc,_config,localCapabilityIndex,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + outp.append((uint8_t)(ccWatch ? 0x16 : 0x02)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -732,11 +718,11 @@ bool Network::filterOutgoingPacket( if ((ztDest != ztFinalDest)&&(ztFinalDest)) { Membership &m2 = _membership(ztFinalDest); - m2.sendCredentialsIfNeeded(RR,now,ztFinalDest,_config,relevantCap); + m2.pushCredentials(RR,now,ztFinalDest,_config,localCapabilityIndex,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x02); // TEE/REDIRECT from outbound side: 0x02 + outp.append((uint8_t)0x04); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -745,11 +731,9 @@ bool Network::filterOutgoingPacket( RR->sw->send(outp,true); return false; // DROP locally, since we redirected - } else if (m) { - m->sendCredentialsIfNeeded(RR,now,ztDest,_config,relevantCap); + } else { + return true; } - - return true; } else { return false; } @@ -765,28 +749,27 @@ int Network::filterIncomingPacket( const unsigned int etherType, const unsigned int vlanId) { - uint32_t remoteTagIds[ZT_MAX_NETWORK_TAGS]; - uint32_t remoteTagValues[ZT_MAX_NETWORK_TAGS]; Address ztFinalDest(ztDest); - Address cc; - unsigned int ccLength = 0; int accept = 0; Mutex::Lock _l(_lock); - Membership &m = _membership(sourcePeer->address()); - const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS); + Membership &membership = _membership(sourcePeer->address()); - switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) { + Address cc; + unsigned int ccLength = 0; + bool ccWatch = false; + switch (_doZtFilter(RR,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { case DOZTFILTER_NO_MATCH: { - Membership::CapabilityIterator mci(m); + Membership::CapabilityIterator mci(membership,_config); const Capability *c; - while ((c = mci.next(_config))) { - ztFinalDest = ztDest; // sanity check + while ((c = mci.next())) { + ztFinalDest = ztDest; // sanity check, should be unmodified if there was no match Address cc2; unsigned int ccLength2 = 0; - switch(_doZtFilter(RR,_config,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc2,ccLength2)) { + bool ccWatch2 = false; + switch(_doZtFilter(RR,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),cc2,ccLength2,ccWatch2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -801,11 +784,11 @@ int Network::filterIncomingPacket( if (accept) { if (cc2) { - _membership(cc2).sendCredentialsIfNeeded(RR,RR->node->now(),cc2,_config,(const Capability *)0); + _membership(cc2).pushCredentials(RR,RR->node->now(),cc2,_config,-1,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + outp.append((uint8_t)(ccWatch2 ? 0x1c : 0x08)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -832,11 +815,11 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { - _membership(cc).sendCredentialsIfNeeded(RR,RR->node->now(),cc,_config,(const Capability *)0); + _membership(cc).pushCredentials(RR,RR->node->now(),cc,_config,-1,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + outp.append((uint8_t)(ccWatch ? 0x1c : 0x08)); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -846,11 +829,11 @@ int Network::filterIncomingPacket( } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { - _membership(ztFinalDest).sendCredentialsIfNeeded(RR,RR->node->now(),ztFinalDest,_config,(const Capability *)0); + _membership(ztFinalDest).pushCredentials(RR,RR->node->now(),ztFinalDest,_config,-1,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); - outp.append((uint8_t)0x06); // TEE/REDIRECT from inbound side: 0x06 + outp.append((uint8_t)0x0a); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); @@ -892,60 +875,6 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) _myMulticastGroups.erase(i); } -bool Network::applyConfiguration(const NetworkConfig &conf) -{ - if (_destroyed) // sanity check - return false; - try { - if ((conf.networkId == _id)&&(conf.issuedTo == RR->identity.address())) { - ZT_VirtualNetworkConfig ctmp; - bool portInitialized; - { - Mutex::Lock _l(_lock); - _config = conf; - _lastConfigUpdate = RR->node->now(); - _netconfFailure = NETCONF_FAILURE_NONE; - _externalConfig(&ctmp); - portInitialized = _portInitialized; - _portInitialized = true; - } - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(portInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); - return true; - } else { - TRACE("ignored invalid configuration for network %.16llx (configuration contains mismatched network ID or issued-to address)",(unsigned long long)_id); - } - } catch (std::exception &exc) { - TRACE("ignored invalid configuration for network %.16llx (%s)",(unsigned long long)_id,exc.what()); - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx (unknown exception)",(unsigned long long)_id); - } - return false; -} - -int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) -{ - try { - { - Mutex::Lock _l(_lock); - if (_config == nconf) - return 1; // OK config, but duplicate of what we already have - } - if (applyConfiguration(nconf)) { - if (saveToDisk) { - char n[64]; - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - Dictionary d; - if (nconf.toDictionary(d,false)) - RR->node->dataStorePut(n,(const void *)d.data(),d.sizeBytes(),true); - } - return 2; // OK and configuration has changed - } - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); - } - return 0; -} - void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize) { std::string newConfig; @@ -979,7 +908,8 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d Identity controllerId(RR->topology->getIdentity(this->controller())); if (controllerId) { if (nc->fromDictionary(*dict)) { - this->setConfiguration(*nc,true); + Mutex::Lock _l(_lock); + this->_setConfiguration(*nc,true); } else { TRACE("error parsing new config with length %u: deserialization of NetworkConfig failed (certificate error?)",(unsigned int)newConfig.length()); } @@ -997,12 +927,6 @@ void Network::handleInboundConfigChunk(const uint64_t inRePacketId,const void *d void Network::requestConfiguration() { - // Sanity limit: do not request more often than once per second - const uint64_t now = RR->node->now(); - if ((now - _lastRequestedConfiguration) < 1000ULL) - return; - _lastRequestedConfiguration = RR->node->now(); - const Address ctrl(controller()); Dictionary rmd; @@ -1024,9 +948,10 @@ void Network::requestConfiguration() if (RR->localNetworkController) { NetworkConfig nconf; switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,nconf)) { - case NetworkController::NETCONF_QUERY_OK: - this->setConfiguration(nconf,true); - return; + case NetworkController::NETCONF_QUERY_OK: { + Mutex::Lock _l(_lock); + this->_setConfiguration(nconf,true); + } return; case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: this->setNotFound(); return; @@ -1073,7 +998,7 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { if (!m) m = &(_membership(peer->address())); - m->sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0); + m->pushCredentials(RR,now,peer->address(),_config,-1,false); if (m->shouldLikeMulticasts(now)) { _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); m->likingMulticasts(now); @@ -1124,9 +1049,8 @@ void Network::clean() Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - if (RR->topology->getPeerNoCache(*a)) - m->clean(_config); - else _memberships.erase(*a); + if (!RR->topology->getPeerNoCache(*a)) + _memberships.erase(*a); } } } @@ -1177,21 +1101,25 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) _sendUpdatesToMembers(&mg); } -int Network::addCredential(const CertificateOfMembership &com) +Membership::AddCredentialResult Network::addCredential(const CertificateOfMembership &com) { if (com.networkId() != _id) - return -1; + return Membership::ADD_REJECTED; const Address a(com.issuedTo()); Mutex::Lock _l(_lock); Membership &m = _membership(a); - const int result = m.addCredential(RR,com); - if (result == 0) { - m.sendCredentialsIfNeeded(RR,RR->node->now(),a,_config,(const Capability *)0); + const Membership::AddCredentialResult result = m.addCredential(RR,_config,com); + if ((result == Membership::ADD_ACCEPTED_NEW)||(result == Membership::ADD_ACCEPTED_REDUNDANT)) { + m.pushCredentials(RR,RR->node->now(),a,_config,-1,false); RR->mc->addCredential(com,true); } return result; } +Membership::AddCredentialResult Network::addCredential(const Revocation &rev) +{ +} + void Network::destroy() { Mutex::Lock _l(_lock); @@ -1215,6 +1143,39 @@ ZT_VirtualNetworkStatus Network::_status() const } } +int Network::_setConfiguration(const NetworkConfig &nconf,bool saveToDisk) +{ + // assumes _lock is locked + try { + if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) + return 0; + if (_config == nconf) + return 1; // OK config, but duplicate of what we already have + + ZT_VirtualNetworkConfig ctmp; + _config = nconf; + _lastConfigUpdate = RR->node->now(); + _netconfFailure = NETCONF_FAILURE_NONE; + _externalConfig(&ctmp); + const bool oldPortInitialized = _portInitialized; + _portInitialized = true; + _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + + if (saveToDisk) { + char n[64]; + Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); + Dictionary d; + if (nconf.toDictionary(d,false)) + RR->node->dataStorePut(n,(const void *)d.data(),d.sizeBytes(),true); + } + + return 2; // OK and configuration has changed + } catch ( ... ) { + TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); + } + return 0; +} + void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const { // assumes _lock is locked @@ -1308,7 +1269,7 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - m->sendCredentialsIfNeeded(RR,now,*a,_config,(const Capability *)0); + m->pushCredentials(RR,now,*a,_config,-1,false); if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { if (!newMulticastGroup) m->likingMulticasts(now); diff --git a/node/Network.hpp b/node/Network.hpp index c85e5993..a151fb88 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -62,6 +62,11 @@ public: */ static const MulticastGroup BROADCAST; + /** + * Compute primary controller device ID from network ID + */ + static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); } + /** * Construct a new network * @@ -76,14 +81,24 @@ public: ~Network(); + inline uint64_t id() const { return _id; } + inline Address controller() const { return Address(_id >> 24); } + inline bool multicastEnabled() const { return (_config.multicastLimit > 0); } + inline bool hasConfig() const { return (_config); } + inline uint64_t lastConfigUpdate() const throw() { return _lastConfigUpdate; } + inline ZT_VirtualNetworkStatus status() const { Mutex::Lock _l(_lock); return _status(); } + inline const NetworkConfig &config() const { return _config; } + inline const MAC &mac() const { return _mac; } + /** * Apply filters to an outgoing packet * * This applies filters from our network config and, if that doesn't match, * our capabilities in ascending order of capability ID. Additional actions - * such as TEE may be taken, and credentials may be pushed. + * such as TEE may be taken, and credentials may be pushed, so this is not + * side-effect-free. It's basically step one in sending something over VL2. * - * @param noTee If true, do not TEE anything anywhere + * @param noTee If true, do not TEE anything anywhere (for two-pass filtering as done with multicast and bridging) * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -134,42 +149,10 @@ public: const unsigned int vlanId); /** - * @return Network ID - */ - inline uint64_t id() const throw() { return _id; } - - /** - * @return Address of network's controller (most significant 40 bits of ID) - */ - inline Address controller() const throw() { return Address(_id >> 24); } - - /** - * @param nwid Network ID - * @return Address of network's controller - */ - static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); } - - /** - * @return Multicast group memberships for this network's port (local, not learned via bridging) - */ - inline std::vector multicastGroups() const - { - Mutex::Lock _l(_lock); - return _myMulticastGroups; - } - - /** - * @return All multicast groups including learned groups that are behind any bridges we're attached to - */ - inline std::vector allMulticastGroups() const - { - Mutex::Lock _l(_lock); - return _allMulticastGroups(); - } - - /** + * Check whether we are subscribed to a multicast group + * * @param mg Multicast group - * @param includeBridgedGroups If true, also include any groups we've learned via bridging + * @param includeBridgedGroups If true, also check groups we've learned via bridging * @return True if this network endpoint / peer is a member */ bool subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBridgedGroups) const; @@ -188,37 +171,19 @@ public: */ void multicastUnsubscribe(const MulticastGroup &mg); - /** - * Apply a NetworkConfig to this network - * - * @param conf Configuration in NetworkConfig form - * @return True if configuration was accepted - */ - bool applyConfiguration(const NetworkConfig &conf); - - /** - * Set or update this network's configuration - * - * @param nconf Network configuration - * @param saveToDisk IF true (default), write config to disk - * @return 0 -- rejected, 1 -- accepted but not new, 2 -- accepted new config - */ - int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); - /** * Handle an inbound network config chunk * - * Only chunks whose inRePacketId matches the packet ID of the last request - * are handled. If this chunk completes the config, it is decoded and - * setConfiguration() is called. + * This is called from IncomingPacket when we receive a chunk from a network + * controller. * - * @param inRePacketId In-re packet ID from OK(NETWORK_CONFIG_REQUEST) + * @param requestId An ID for grouping chunks, e.g. in-re packet ID for OK(NETWORK_CONFIG_REQUEST) * @param data Chunk data * @param chunkSize Size of data[] * @param chunkIndex Index of chunk in full config * @param totalSize Total size of network config */ - void handleInboundConfigChunk(const uint64_t inRePacketId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize); + void handleInboundConfigChunk(const uint64_t requestId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize); /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this @@ -230,7 +195,7 @@ public: } /** - * Set netconf failure to 'not found' -- called by PacketDecider when controller reports this + * Set netconf failure to 'not found' -- called by IncomingPacket when controller reports this */ inline void setNotFound() { @@ -240,10 +205,6 @@ public: /** * Causes this network to request an updated configuration from its master node now - * - * There is a circuit breaker here to prevent this from being done more often - * than once per second. This is to prevent things like NETWORK_CONFIG_REFRESH - * from causing multiple requests. */ void requestConfiguration(); @@ -251,7 +212,7 @@ public: * Determine whether this peer is permitted to communicate on this network * * This also performs certain periodic actions such as pushing renewed - * credentials to peers or requesting them if not present. + * credentials to peers, so like the filters it is not side-effect-free. * * @param peer Peer to check * @param verb Packet verb @@ -266,7 +227,7 @@ public: bool gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); /** - * Perform cleanup and possibly save state + * Do periodic cleanup and housekeeping tasks */ void clean(); @@ -279,46 +240,6 @@ public: _sendUpdatesToMembers((const MulticastGroup *)0); } - /** - * @return Time of last updated configuration or 0 if none - */ - inline uint64_t lastConfigUpdate() const throw() { return _lastConfigUpdate; } - - /** - * @return Status of this network - */ - inline ZT_VirtualNetworkStatus status() const - { - Mutex::Lock _l(_lock); - return _status(); - } - - /** - * @param ec Buffer to fill with externally-visible network configuration - */ - inline void externalConfig(ZT_VirtualNetworkConfig *ec) const - { - Mutex::Lock _l(_lock); - _externalConfig(ec); - } - - /** - * Get current network config - * - * @return Network configuration (may be a null config if we don't have one yet) - */ - inline const NetworkConfig &config() const { return _config; } - - /** - * @return True if this network has a valid config - */ - inline bool hasConfig() const { return (_config); } - - /** - * @return Ethernet MAC address for this network's local interface - */ - inline const MAC &mac() const { return _mac; } - /** * Find the node on this network that has this MAC behind it (if any) * @@ -349,44 +270,47 @@ public: void learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now); /** - * @param com Certificate of membership - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + * Validate a credential and learn it if it passes certificate and other checks */ - int addCredential(const CertificateOfMembership &com); + Membership::AddCredentialResult addCredential(const CertificateOfMembership &com); /** - * @param cap Capability - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + * Validate a credential and learn it if it passes certificate and other checks */ - inline int addCredential(const Capability &cap) + inline Membership::AddCredentialResult addCredential(const Capability &cap) { if (cap.networkId() != _id) - return -1; + return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(cap.issuedTo()).addCredential(RR,cap); + return _membership(cap.issuedTo()).addCredential(RR,_config,cap); } /** - * @param cap Tag - * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + * Validate a credential and learn it if it passes certificate and other checks */ - inline int addCredential(const Tag &tag) + inline Membership::AddCredentialResult addCredential(const Tag &tag) { if (tag.networkId() != _id) - return -1; + return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(tag.issuedTo()).addCredential(RR,tag); + return _membership(tag.issuedTo()).addCredential(RR,_config,tag); } /** - * Blacklist COM, tags, and capabilities before this time + * Validate a credential and learn it if it passes certificate and other checks + */ + Membership::AddCredentialResult addCredential(const Revocation &rev); + + /** + * Force push credentials (COM, etc.) to a peer now * - * @param ts Blacklist cutoff + * @param to Destination peer address + * @param now Current time */ - inline void blacklistBefore(const Address &peerAddress,const uint64_t ts) + inline void pushCredentialsNow(const Address &to,const uint64_t now) { Mutex::Lock _l(_lock); - _membership(peerAddress).blacklistBefore(ts); + _membership(to).pushCredentials(RR,now,to,_config,-1,true); } /** @@ -399,11 +323,23 @@ public: void destroy(); /** - * @return Pointer to user PTR (modifiable user ptr used in API) + * Get this network's config for export via the ZT core API + * + * @param ec Buffer to fill with externally-visible network configuration + */ + inline void externalConfig(ZT_VirtualNetworkConfig *ec) const + { + Mutex::Lock _l(_lock); + _externalConfig(ec); + } + + /** + * @return Externally usable pointer-to-pointer exported via the core API */ inline void **userPtr() throw() { return &_uPtr; } private: + int _setConfiguration(const NetworkConfig &nconf,bool saveToDisk); ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); @@ -412,9 +348,9 @@ private: std::vector _allMulticastGroups() const; Membership &_membership(const Address &a); - const RuntimeEnvironment *RR; + const RuntimeEnvironment *const RR; void *_uPtr; - uint64_t _id; + const uint64_t _id; uint64_t _lastAnnouncedMulticastGroupsUpstream; MAC _mac; // local MAC address volatile bool _portInitialized; @@ -428,7 +364,6 @@ private: NetworkConfig _config; volatile uint64_t _lastConfigUpdate; - volatile uint64_t _lastRequestedConfiguration; volatile bool _destroyed; diff --git a/node/Packet.hpp b/node/Packet.hpp index 2ca73a84..03b9b113 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -655,15 +655,27 @@ public: * * Flags: * 0x01 - Certificate of network membership attached (DEPRECATED) - * 0x02 - This is a TEE'd or REDIRECT'ed packet - * 0x04 - TEE/REDIRECT'ed packet is from inbound side - * + * 0x02 - Most significant bit of subtype (see below) + * 0x04 - Middle bit of subtype (see below) + * 0x08 - Least significant bit of subtype (see below) + * 0x10 - ACK requested in the form of OK(EXT_FRAME) + * + * Subtypes (0..7): + * 0x0 - Normal frame (bridging can be determined by checking MAC) + * 0x1 - TEEd outbound frame + * 0x2 - REDIRECTed outbound frame + * 0x3 - WATCHed outbound frame (TEE with ACK, ACK bit also set) + * 0x4 - TEEd inbound frame + * 0x5 - REDIRECTed inbound frame + * 0x6 - WATCHed inbound frame + * 0x7 - (reserved for future use) + * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we * want to attach a certificate since FRAME does not support that. * - * ERROR may be generated if a membership certificate is needed for a - * closed network. Payload will be network ID. + * If the ACK flag (0x08) is set, an OK(EXT_FRAME) is sent with + * no payload to acknowledge receipt of the frame. */ VERB_EXT_FRAME = 0x07, @@ -698,7 +710,7 @@ public: VERB_MULTICAST_LIKE = 0x09, /** - * Network membership credential push: + * Network credentials push: * <[...] serialized certificate of membership> * [<[...] additional certificates of membership>] * <[1] 0x00, null byte marking end of COM array> @@ -706,12 +718,12 @@ public: * <[...] one or more serialized Capability> * <[2] 16-bit number of tags> * <[...] one or more serialized Tags> + * <[2] 16-bit number of revocations> + * <[...] one or more serialized Revocations> * - * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may - * be pushed at any other time to keep exchanged certificates up to date. - * - * COMs and other credentials need not be for the same network, since each - * includes its own network ID and signature. + * This can be sent by anyone at any time to push network credentials. + * These will of course only be accepted if they are properly signed. + * Credentials can be for any number of networks. * * OK/ERROR are not generated. */ @@ -742,23 +754,18 @@ public: VERB_NETWORK_CONFIG_REQUEST = 0x0b, /** - * Network configuration update push: - * <[8] network ID to refresh> - * <[2] 16-bit number of address/timestamp pairs to blacklist> - * [<[5] ZeroTier address of peer being revoked>] - * [<[8] blacklist credentials older than this timestamp>] - * [<[...] additional address/timestamp pairs>] - * - * This can be sent by a network controller to both request that a network - * config be updated and push instantaneous revocations of specific peers - * or peer credentials. - * - * Specific revocations can be pushed to blacklist a specific peer's - * credentials (COM, tags, and capabilities) if older than a specified - * timestamp. This can be used to accomplish expedited revocation of - * a peer's access to things on a network or to the network itself among - * those other peers that can currently reach the controller. This is not - * the only mechanism for revocation of course, but it's the fastest. + * Network configuration push: + * <[8] 64-bit network ID> + * <[8] 64-bit value used to group chunks in this push> + * <[2] 16-bit length of network configuration dictionary chunk> + * <[...] network configuration dictionary (may be incomplete)> + * <[4] 32-bit total length of assembled dictionary> + * <[4] 32-bit index of chunk in this reply> + * + * This is a direct push variant for network config updates. It otherwise + * carries the same payload as OK(NETWORK_CONFIG_REQUEST). There is an + * extra number after network ID in this version that is used in place of + * the in-re packet ID sent with OKs to group chunks together. */ VERB_NETWORK_CONFIG_REFRESH = 0x0c, diff --git a/node/Revocation.cpp b/node/Revocation.cpp new file mode 100644 index 00000000..420476a4 --- /dev/null +++ b/node/Revocation.cpp @@ -0,0 +1,46 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "Revocation.hpp" +#include "RuntimeEnvironment.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Switch.hpp" +#include "Network.hpp" + +namespace ZeroTier { + +int Revocation::verify(const RuntimeEnvironment *RR) const +{ + if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) + return -1; + const Identity id(RR->topology->getIdentity(_signedBy)); + if (!id) { + RR->sw->requestWhois(_signedBy); + return 1; + } + try { + Buffer tmp; + this->serialize(tmp,true); + return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1); + } catch ( ... ) { + return -1; + } +} + +} // namespace ZeroTier diff --git a/node/Revocation.hpp b/node/Revocation.hpp new file mode 100644 index 00000000..58757465 --- /dev/null +++ b/node/Revocation.hpp @@ -0,0 +1,178 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_REVOCATION_HPP +#define ZT_REVOCATION_HPP + +#include +#include +#include +#include + +#include "Constants.hpp" +#include "../include/ZeroTierOne.h" +#include "Address.hpp" +#include "C25519.hpp" +#include "Utils.hpp" +#include "Buffer.hpp" +#include "Identity.hpp" + +/** + * Flag: fast propagation via rumor mill algorithm + */ +#define ZT_REVOCATION_FLAG_FAST_PROPAGATE 0x1ULL + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * Revocation certificate to instantaneously revoke a COM, capability, or tag + */ +class Revocation +{ +public: + enum CredentialType + { + CREDENTIAL_TYPE_NIL = 0, + CREDENTIAL_TYPE_COM = 1, + CREDENTIAL_TYPE_CAPABILITY = 2, + CREDENTIAL_TYPE_TAG = 3 + }; + + Revocation() + { + memset(this,0,sizeof(Revocation)); + } + + Revocation(const uint64_t i,const uint64_t nwid,const uint64_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const CredentialType ct) : + _id(i), + _networkId(nwid), + _credentialId(cid), + _threshold(thr), + _flags(fl), + _target(tgt), + _signedBy(), + _type(ct) {} + + inline uint64_t id() const { return _id; } + inline uint64_t networkId() const { return _networkId; } + inline uint64_t credentialId() const { return _credentialId; } + inline uint64_t threshold() const { return _threshold; } + inline const Address &target() const { return _target; } + inline const Address &signer() const { return _signedBy; } + inline CredentialType type() const { return _type; } + + inline bool fastPropagate() const { return ((_flags & ZT_REVOCATION_FLAG_FAST_PROPAGATE) != 0); } + + /** + * @param signer Signing identity, must have private key + * @return True if signature was successful + */ + inline bool sign(const Identity &signer) + { + if (signer.hasPrivate()) { + Buffer tmp; + this->serialize(tmp,true); + _signedBy = signer.address(); + _signature = signer.sign(tmp.data(),tmp.size()); + return true; + } + return false; + } + + /** + * Verify this revocation's signature + * + * @param RR Runtime environment to provide for peer lookup, etc. + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain + */ + int verify(const RuntimeEnvironment *RR) const; + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_id); + b.append(_networkId); + b.append(_credentialId); + b.append(_threshold); + b.append(_flags); + _target.appendTo(b); + _signedBy.appendTo(b); + b.append((uint8_t)_type); + + if (!forSign) { + b.append((uint8_t)1); // 1 == Ed25519 signature + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); + } + + // This is the size of any additional fields, currently 0. + b.append((uint16_t)0); + + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + memset(this,0,sizeof(Revocation)); + + unsigned int p = startAt; + + _id = b.template at(p); p += 8; + _networkId = b.template at(p); p += 8; + _credentialId = b.template at(p); p += 8; + _threshold = b.template at(p); p += 8; + _flags = b.template at(p); p += 8; + _target.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + _type = (CredentialType)b[p++]; + + if (b[p++] == 1) { + if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); + p += ZT_C25519_SIGNATURE_LEN; + } else throw std::runtime_error("invalid signature"); + } + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + +private: + uint64_t _id; + uint64_t _networkId; + uint64_t _credentialId; + uint64_t _threshold; + uint64_t _flags; + Address _target; + Address _signedBy; + CredentialType _type; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/Tag.cpp b/node/Tag.cpp index 352ecde8..eb4026bc 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -27,7 +27,7 @@ namespace ZeroTier { int Tag::verify(const RuntimeEnvironment *RR) const { - if ((!_signedBy)||(_signedBy != Network::controllerFor(_nwid))) + if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; const Identity id(RR->topology->getIdentity(_signedBy)); if (!id) { diff --git a/node/Tag.hpp b/node/Tag.hpp index 14cc3a5d..97228157 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -67,7 +67,7 @@ public: * @param value Tag value */ Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : - _nwid(nwid), + _networkId(nwid), _ts(ts), _id(id), _value(value), @@ -76,7 +76,7 @@ public: { } - inline uint64_t networkId() const { return _nwid; } + inline uint64_t networkId() const { return _networkId; } inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } @@ -91,13 +91,13 @@ public: */ inline bool sign(const Identity &signer) { - try { - Buffer<(sizeof(Tag) * 2)> tmp; + if (signer.hasPrivate()) { + Buffer tmp; _signedBy = signer.address(); this->serialize(tmp,true); _signature = signer.sign(tmp.data(),tmp.size()); return true; - } catch ( ... ) {} + } return false; } @@ -115,7 +115,7 @@ public: if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); // These are the same between Tag and Capability - b.append(_nwid); + b.append(_networkId); b.append(_ts); b.append(_id); @@ -140,7 +140,7 @@ public: unsigned int p = startAt; // These are the same between Tag and Capability - _nwid = b.template at(p); p += 8; + _networkId = b.template at(p); p += 8; _ts = b.template at(p); p += 8; _id = b.template at(p); p += 4; @@ -168,8 +168,22 @@ public: inline bool operator==(const Tag &t) const { return (memcmp(this,&t,sizeof(Tag)) == 0); } inline bool operator!=(const Tag &t) const { return (memcmp(this,&t,sizeof(Tag)) != 0); } + // For searching sorted arrays or lists of Tags by ID + struct IdComparePredicate + { + inline bool operator()(const Tag &a,const Tag &b) const { return (a.id() < b.id()); } + inline bool operator()(const uint32_t a,const Tag &b) const { return (a < b.id()); } + inline bool operator()(const Tag &a,const uint32_t b) const { return (a.id() < b); } + inline bool operator()(const Tag *a,const Tag *b) const { return (a->id() < b->id()); } + inline bool operator()(const Tag *a,const Tag &b) const { return (a->id() < b.id()); } + inline bool operator()(const Tag &a,const Tag *b) const { return (a.id() < b->id()); } + inline bool operator()(const uint32_t a,const Tag *b) const { return (a < b->id()); } + inline bool operator()(const Tag *a,const uint32_t b) const { return (a->id() < b); } + inline bool operator()(const uint32_t a,const uint32_t b) const { return (a < b); } + }; + private: - uint64_t _nwid; + uint64_t _networkId; uint64_t _ts; uint32_t _id; uint32_t _value; diff --git a/objects.mk b/objects.mk index f92a907e..5738e769 100644 --- a/objects.mk +++ b/objects.mk @@ -17,6 +17,7 @@ OBJS=\ node/Path.o \ node/Peer.o \ node/Poly1305.o \ + node/Revocation.o \ node/Salsa20.o \ node/SelfAwareness.o \ node/SHA512.o \ -- cgit v1.2.3 From eac3667ec1391baaf83c64403e8a19c22c24c9f2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 26 Sep 2016 16:17:02 -0700 Subject: Bunch more refactoring and work on revocations, etc. --- include/ZeroTierOne.h | 2 +- node/IncomingPacket.cpp | 53 ++++++++++++------ node/IncomingPacket.hpp | 2 +- node/Membership.cpp | 143 ++++++++++++++++++++++++++++++++++++------------ node/Membership.hpp | 21 +++++-- node/Network.cpp | 37 ++++++++++++- node/Network.hpp | 2 +- node/Packet.cpp | 2 +- node/Packet.hpp | 28 +++++++--- node/Revocation.hpp | 2 +- 10 files changed, 220 insertions(+), 72 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e43c8541..591ff1fe 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -517,7 +517,7 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_ACTION_TEE = 2, /** - * Exactly like TEE but frames are dropped if previous TEEs were not acknowledged by the observer + * Exactly like TEE but mandates ACKs from observer */ ZT_NETWORK_RULE_ACTION_WATCH = 3, diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 12766fe2..c50db794 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -67,7 +67,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) return _doHELLO(RR,false); } - SharedPtr peer(RR->topology->getPeer(sourceAddress)); + const SharedPtr peer(RR->topology->getPeer(sourceAddress)); if (peer) { if (!trusted) { if (!dearmor(peer->key())) { @@ -100,7 +100,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); - case Packet::VERB_NETWORK_CONFIG_REFRESH: return _doNETWORK_CONFIG_REFRESH(RR,peer); + case Packet::VERB_NETWORK_CONFIG: return _doNETWORK_CONFIG(RR,peer); case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); @@ -131,12 +131,18 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr //TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); + /* Security note: we do not gate doERROR() with expectingReplyTo() to + * avoid having to log every outgoing packet ID. Instead we put the + * logic to determine whether we should consider an ERROR in each + * error handler. In most cases these are only trusted in specific + * circumstances. */ + switch(errorCode) { case Packet::ERROR_OBJ_NOT_FOUND: // Object not found, currently only meaningful from network controllers. if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) network->setNotFound(); } @@ -147,7 +153,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // consider it meaningful from network controllers. This would indicate // that the queried node does not support acting as a controller. if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) network->setNotFound(); } @@ -161,7 +167,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { // Peers can send this in response to frames if they do not have a recent enough COM from us - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); const uint64_t now = RR->node->now(); if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) network->pushCredentialsNow(peer->address(),now); @@ -169,7 +175,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_NETWORK_ACCESS_DENIED_: { // Network controller: network access denied. - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) network->setAccessDenied(); } break; @@ -177,9 +183,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_UNWANTED_MULTICAST: { // Members of networks can use this error to indicate that they no longer // want to receive multicasts on a given channel. - SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->gate(peer,verb(),packetId()))) { - MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); RR->mc->remove(network->id(),mg,peer->address()); } @@ -371,7 +377,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); bool trustEstablished = false; - // Don't parse OK packets that are not in response to a packet ID we sent if (!RR->node->expectingReplyTo(inRePacketId)) { TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId()); return true; @@ -450,7 +455,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); - SharedPtr network(RR->node->network(nwid)); + const SharedPtr network(RR->node->network(nwid)); if ((network)&&(network->gateMulticastGatherReply(peer,verb(),packetId()))) { trustEstablished = true; const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); @@ -467,7 +472,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p //TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags); - SharedPtr network(RR->node->network(nwid)); + const SharedPtr network(RR->node->network(nwid)); if (network) { unsigned int offset = 0; @@ -683,6 +688,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

address(),RR->identity.address(),Packet::VERB_OK); outp.append((uint8_t)Packet::VERB_EXT_FRAME); outp.append((uint64_t)packetId()); + outp.append((uint64_t)nwid); outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } @@ -727,7 +733,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared try { const uint64_t now = RR->node->now(); - uint64_t authOnNetwork[256]; + uint64_t authOnNetwork[256]; // cache for approved network IDs unsigned int authOnNetworkCount = 0; SharedPtr network; bool trustEstablished = false; @@ -786,7 +792,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S while ((p < size())&&((*this)[p])) { p += com.deserialize(*this,p); if (com) { - SharedPtr network(RR->node->network(com.networkId())); + const SharedPtr network(RR->node->network(com.networkId())); if (network) { switch (network->addCredential(com)) { case Membership::ADD_REJECTED: @@ -803,11 +809,11 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } ++p; // skip trailing 0 after COMs if present - if (p < size()) { // check if new capabilities and tags fields are present + if (p < size()) { // older ZeroTier versions do not send capabilities, tags, or revocations const unsigned int numCapabilities = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(cap.networkId())); + const SharedPtr network(RR->node->network(cap.networkId())); if (network) { switch (network->addCredential(cap)) { case Membership::ADD_REJECTED: @@ -825,7 +831,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(tag.networkId())); + const SharedPtr network(RR->node->network(tag.networkId())); if (network) { switch (network->addCredential(tag)) { case Membership::ADD_REJECTED: @@ -843,8 +849,18 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S const unsigned int numRevocations = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(revocation.networkId())); + const SharedPtr network(RR->node->network(revocation.networkId())); if (network) { + switch(network->addCredential(peer->address(),revocation)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; + } } } } @@ -879,6 +895,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons try { if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { dconf->wrapWithSignature(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE,RR->identity.privateKeyPair()); + const unsigned int totalSize = dconf->sizeBytes(); unsigned int chunkIndex = 0; while (chunkIndex < totalSize) { @@ -957,7 +974,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons return true; } -bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PACKET_IDX_PAYLOAD); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index dbaf67b8..86c2b5e7 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -146,7 +146,7 @@ private: bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,const SharedPtr &peer); + bool _doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); diff --git a/node/Membership.cpp b/node/Membership.cpp index d579d303..b7e33936 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -167,24 +167,9 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme return ADD_REJECTED; case 0: TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (have) { - have->lastReceived = RR->node->now(); - have->tag = tag; - } else { - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == 0xffffffffffffffffULL) { - have = _remoteTags[i]; - break; - } else if (_remoteTags[i]->lastReceived <= minlr) { - have = _remoteTags[i]; - minlr = _remoteTags[i]->lastReceived; - } - } - have->lastReceived = RR->node->now(); - have->tag = tag; - std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); - } + if (!have) have = _newTag(tag.id()); + have->lastReceived = RR->node->now(); + have->tag = tag; return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -212,28 +197,114 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme return ADD_REJECTED; case 0: TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (have) { - have->lastReceived = RR->node->now(); - have->cap = cap; - } else { - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == 0xffffffffffffffffULL) { - have = _remoteCaps[i]; - break; - } else if (_remoteCaps[i]->lastReceived <= minlr) { - have = _remoteCaps[i]; - minlr = _remoteCaps[i]->lastReceived; - } - } - have->lastReceived = RR->node->now(); - have->cap = cap; - std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); - } + if (!have) have = _newCapability(cap.id()); + have->lastReceived = RR->node->now(); + have->cap = cap; return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; } } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev) +{ + switch(rev.verify(RR)) { + default: + return ADD_REJECTED; + case 0: { + const uint64_t now = RR->node->now(); + switch(rev.type()) { + default: + //case Revocation::CREDENTIAL_TYPE_ALL: + return ( (_revokeCom(rev)||_revokeCap(rev,now)||_revokeTag(rev,now)) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT ); + case Revocation::CREDENTIAL_TYPE_COM: + return (_revokeCom(rev) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); + case Revocation::CREDENTIAL_TYPE_CAPABILITY: + return (_revokeCap(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); + case Revocation::CREDENTIAL_TYPE_TAG: + return (_revokeTag(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); + } + } + case 1: + return ADD_DEFERRED_FOR_WHOIS; + } +} + +Membership::_RemoteTag *Membership::_newTag(const uint64_t id) +{ + _RemoteTag *t; + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { + t = _remoteTags[i]; + break; + } else if (_remoteTags[i]->lastReceived <= minlr) { + t = _remoteTags[i]; + minlr = _remoteTags[i]->lastReceived; + } + } + t->id = id; + t->lastReceived = 0; + t->revocationThreshold = 0; + t->tag = Tag(); + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); + return t; +} + +Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) +{ + _RemoteCapability *c; + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { + c = _remoteCaps[i]; + break; + } else if (_remoteCaps[i]->lastReceived <= minlr) { + c = _remoteCaps[i]; + minlr = _remoteCaps[i]->lastReceived; + } + } + c->id = id; + c->lastReceived = 0; + c->revocationThreshold = 0; + c->cap = Capability(); + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); + return c; +} + +bool Membership::_revokeCom(const Revocation &rev) +{ + if (rev.threshold() > _comRevocationThreshold) { + _comRevocationThreshold = rev.threshold(); + return true; + } + return false; +} + +bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) +{ + _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialSorter<_RemoteCapability>()); + _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCapability *)0; + if (!have) have = _newCapability(rev.credentialId()); + if (rev.threshold() > have->revocationThreshold) { + have->lastReceived = now; + have->revocationThreshold = rev.threshold(); + return true; + } + return false; +} + +bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) +{ + _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialSorter<_RemoteTag>()); + _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteTag *)0; + if (!have) have = _newTag(rev.credentialId()); + if (rev.threshold() > have->revocationThreshold) { + have->lastReceived = now; + have->revocationThreshold = rev.threshold(); + return true; + } + return false; +} + } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 421e3ee8..c54aec9b 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -29,6 +29,8 @@ #include "Revocation.hpp" #include "NetworkConfig.hpp" +#define ZT_MEMBERSHIP_CRED_ID_UNUSED 0xffffffffffffffffULL + namespace ZeroTier { class RuntimeEnvironment; @@ -48,7 +50,7 @@ private: // Tags and related state struct _RemoteTag { - _RemoteTag() : id(0xffffffffffffffffULL),lastReceived(0),revocationThreshold(0) {} + _RemoteTag() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} // Tag ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) uint64_t id; // Last time we received THEIR tag (with this ID) @@ -62,7 +64,7 @@ private: // Credentials and related state struct _RemoteCapability { - _RemoteCapability() : id(0xffffffffffffffffULL),lastReceived(0),revocationThreshold(0) {} + _RemoteCapability() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} // Capability ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) uint64_t id; // Last time we received THEIR capability (with this ID) @@ -114,7 +116,7 @@ public: inline const Capability *next() { for(;;) { - if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != 0xffffffffffffffffULL)) { + if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { const Capability *tmp = &((*_i)->cap); if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { ++_i; @@ -147,7 +149,7 @@ public: inline const Tag *next() { for(;;) { - if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != 0xffffffffffffffffULL)) { + if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { const Tag *tmp = &((*_i)->tag); if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { ++_i; @@ -242,7 +244,18 @@ public: */ AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap); + /** + * Validate and add a credential if signature is okay and it's otherwise good + */ + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev); + private: + _RemoteTag *_newTag(const uint64_t id); + _RemoteCapability *_newCapability(const uint64_t id); + bool _revokeCom(const Revocation &rev); + bool _revokeCap(const Revocation &rev,const uint64_t now); + bool _revokeTag(const Revocation &rev,const uint64_t now); + template inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred,const CS &state) const { diff --git a/node/Network.cpp b/node/Network.cpp index 0fab6a27..487766a7 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -36,7 +36,7 @@ #include "Peer.hpp" // Uncomment to make the rules engine dump trace info to stdout -#define ZT_RULES_ENGINE_DEBUGGING 1 +//#define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { @@ -1116,8 +1116,41 @@ Membership::AddCredentialResult Network::addCredential(const CertificateOfMember return result; } -Membership::AddCredentialResult Network::addCredential(const Revocation &rev) +Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,const Revocation &rev) { + if (rev.networkId() != _id) + return Membership::ADD_REJECTED; + + Mutex::Lock _l(_lock); + Membership &m = _membership(rev.target()); + + const Membership::AddCredentialResult result = m.addCredential(RR,_config,rev); + + if ((result == Membership::ADD_ACCEPTED_NEW)&&(rev.fastPropagate())) { + /* Fast propagation is done by using a very aggressive rumor mill + * propagation algorithm. When we see a Revocation that we haven't + * seen before we blast it to every known member. This leads to + * a huge number of redundant messages, but eventually everybody + * will get it. This helps revocation speed and also helps in cases + * where the controller is under attack. It need only get one + * revocation out and the rest is history. */ + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) { + if ((*a != sentFrom)&&(*a != rev.signer())) { + Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + outp.append((uint8_t)0x00); // no COM + outp.append((uint16_t)0); // no capabilities + outp.append((uint16_t)0); // no tags + outp.append((uint16_t)1); // one revocation! + rev.serialize(outp); + RR->sw->send(outp,true); + } + } + } + + return result; } void Network::destroy() diff --git a/node/Network.hpp b/node/Network.hpp index a151fb88..6a1ac801 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -299,7 +299,7 @@ public: /** * Validate a credential and learn it if it passes certificate and other checks */ - Membership::AddCredentialResult addCredential(const Revocation &rev); + Membership::AddCredentialResult addCredential(const Address &sentFrom,const Revocation &rev); /** * Force push credentials (COM, etc.) to a peer now diff --git a/node/Packet.cpp b/node/Packet.cpp index 9ab68968..3b8e1387 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -40,7 +40,7 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; case VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; - case VERB_NETWORK_CONFIG_REFRESH: return "NETWORK_CONFIG_REFRESH"; + case VERB_NETWORK_CONFIG: return "NETWORK_CONFIG_REFRESH"; case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; diff --git a/node/Packet.hpp b/node/Packet.hpp index 03b9b113..e76cb96c 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -674,8 +674,8 @@ public: * superset of VERB_FRAME. They're used for bridging or when we * want to attach a certificate since FRAME does not support that. * - * If the ACK flag (0x08) is set, an OK(EXT_FRAME) is sent with - * no payload to acknowledge receipt of the frame. + * OK payload (if ACK flag is set): + * <[8] 64-bit network ID> */ VERB_EXT_FRAME = 0x07, @@ -738,9 +738,14 @@ public: * <[8] 64-bit timestamp of netconf we currently have> * * This message requests network configuration from a node capable of - * providing it. If the optional revision is included, a response is - * only generated if there is a newer network configuration available. + * providing it. + * + * Respones to this are always whole configs intended for the recipient. + * For patches and other updates a NETWORK_CONFIG is sent instead. * + * It would be valid and correct as of 1.2.0 to use NETWORK_CONFIG always, + * but OK(NTEWORK_CONFIG_REQUEST) should be sent for compatibility. + * * OK response payload: * <[8] 64-bit network ID> * <[2] 16-bit length of network configuration dictionary chunk> @@ -754,9 +759,10 @@ public: VERB_NETWORK_CONFIG_REQUEST = 0x0b, /** - * Network configuration push: + * Network configuration data push: * <[8] 64-bit network ID> - * <[8] 64-bit value used to group chunks in this push> + * <[8] 64-bit config update ID (token to identify this update)> + * <[1] flags> * <[2] 16-bit length of network configuration dictionary chunk> * <[...] network configuration dictionary (may be incomplete)> * <[4] 32-bit total length of assembled dictionary> @@ -766,8 +772,16 @@ public: * carries the same payload as OK(NETWORK_CONFIG_REQUEST). There is an * extra number after network ID in this version that is used in place of * the in-re packet ID sent with OKs to group chunks together. + * + * Unlike OK(NETWORK_CONFIG_REQUEST) this can be sent by peers other than + * network controllers. In that case the certificate inside the Dictionary + * is used for verification purposes. + * + * Flags: + * 0x01 - Patch, not whole config + * 0x02 - Use fast P2P propagation */ - VERB_NETWORK_CONFIG_REFRESH = 0x0c, + VERB_NETWORK_CONFIG = 0x0c, /** * Request endpoints for multicast distribution: diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 58757465..18916985 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -49,7 +49,7 @@ class Revocation public: enum CredentialType { - CREDENTIAL_TYPE_NIL = 0, + CREDENTIAL_TYPE_ALL = 0, CREDENTIAL_TYPE_COM = 1, CREDENTIAL_TYPE_CAPABILITY = 2, CREDENTIAL_TYPE_TAG = 3 -- cgit v1.2.3 From 7e4b6b594b9529565b8bb3acb6d99e37c1f3db1b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 26 Sep 2016 17:05:39 -0700 Subject: It now builds. --- controller/EmbeddedNetworkController.cpp | 4 ++-- include/ZeroTierOne.h | 21 --------------------- node/IncomingPacket.cpp | 2 +- node/Node.cpp | 32 -------------------------------- node/Node.hpp | 1 - node/Packet.hpp | 11 ++++++++--- 6 files changed, 11 insertions(+), 60 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5ba8cf98..cd8ce8bf 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -516,8 +516,8 @@ void EmbeddedNetworkController::threadMain() Mutex::Lock _l(_refreshQueue_m); while (_refreshQueue.size() > 0) { _Refresh &r = _refreshQueue.front(); - if (_node) - _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); + //if (_node) + // _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); _refreshQueue.pop_front(); if (++count >= 50) break; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 591ff1fe..c66b9079 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1928,27 +1928,6 @@ enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,v */ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test); -/** - * Push a network refresh - * - * This is used by network controller implementations to send a - * NETWORK_CONFIG_REFRESH message to tell a node to refresh its - * config and to optionally push one or more credential timestamp - * blacklist thresholds for members of the network. - * - * Code outside a controller implementation will have no use for - * this as these messages are ignored if they do not come from a - * controller. - * - * @param node Node instance - * @param dest ZeroTier address of destination to which to send NETWORK_CONFIG_REFRESH - * @param nwid Network ID - * @param blacklistAddresses Array of ZeroTier addresses of network members to set timestamp blacklists for - * @param blacklistBeforeTimestamps Timestamps before which to blacklist credentials for each corresponding address in blacklistAddresses[] - * @param blacklistCount Size of blacklistAddresses[] and blacklistBeforeTimestamps[] - */ -void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); - /** * Initialize cluster operation * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c50db794..72dfbfd8 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -982,7 +982,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,trustEstablished); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } diff --git a/node/Node.cpp b/node/Node.cpp index 2533eeb6..db9b8ea0 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -552,31 +552,6 @@ void Node::circuitTestEnd(ZT_CircuitTest *test) } } -void Node::pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) -{ - Packet outp(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); - outp.append(nwid); - outp.addSize(2); - unsigned int c = 0; - for(unsigned int i=0;i= ZT_PROTO_MAX_PACKET_LENGTH) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); - RR->sw->send(outp,true); - outp = Packet(Address(dest),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REFRESH); - outp.append(nwid); - outp.addSize(2); - c = 0; - } - Address(blacklistAddresses[i]).appendTo(outp); - outp.append(blacklistBeforeTimestamps[i]); - ++c; - } - if (c > 0) { - outp.setAt(ZT_PACKET_IDX_PAYLOAD + 8,(uint16_t)c); - RR->sw->send(outp,true); - } -} - ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -973,13 +948,6 @@ void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test) } catch ( ... ) {} } -void ZT_Node_pushNetworkRefresh(ZT_Node *node,uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount) -{ - try { - reinterpret_cast(node)->pushNetworkRefresh(dest,nwid,blacklistAddresses,blacklistBeforeTimestamps,blacklistCount); - } catch ( ... ) {} -} - enum ZT_ResultCode ZT_Node_clusterInit( ZT_Node *node, unsigned int myId, diff --git a/node/Node.hpp b/node/Node.hpp index 56869816..11462531 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -107,7 +107,6 @@ public: void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); - void pushNetworkRefresh(uint64_t dest,uint64_t nwid,const uint64_t *blacklistAddresses,const uint64_t *blacklistBeforeTimestamps,unsigned int blacklistCount); ZT_ResultCode clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, diff --git a/node/Packet.hpp b/node/Packet.hpp index e76cb96c..b03ec327 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -670,9 +670,11 @@ public: * 0x6 - WATCHed inbound frame * 0x7 - (reserved for future use) * - * An extended frame carries full MAC addressing, making them a - * superset of VERB_FRAME. They're used for bridging or when we - * want to attach a certificate since FRAME does not support that. + * An extended frame carries full MAC addressing, making it a + * superset of VERB_FRAME. It is used for bridged traffic, + * redirected or observed traffic via rules, and can in theory + * be used for multicast though MULTICAST_FRAME exists for that + * purpose and has additional options and capabilities. * * OK payload (if ACK flag is set): * <[8] 64-bit network ID> @@ -725,6 +727,9 @@ public: * These will of course only be accepted if they are properly signed. * Credentials can be for any number of networks. * + * The use of a zero byte to terminate the COM section is for legacy + * backward compatiblity. Newer fields are prefixed with a length. + * * OK/ERROR are not generated. */ VERB_NETWORK_CREDENTIALS = 0x0a, -- cgit v1.2.3 From 15c07c58b610f699fd2a7164fde96712e1595f2b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Sep 2016 11:33:48 -0700 Subject: Refactored network config chunking to sign every chunk to prevent stupid DOS attack potential, and implement network config fast propagate (though we probably will not use this for a bit). --- node/Dictionary.hpp | 42 +----------- node/IncomingPacket.cpp | 54 +++++++++------- node/Network.cpp | 167 ++++++++++++++++++++++++++++++++++-------------- node/Network.hpp | 40 +++++++----- node/Packet.hpp | 49 ++++++++++---- 5 files changed, 214 insertions(+), 138 deletions(-) (limited to 'node') diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index eab2b162..15ab9ce3 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -23,7 +23,6 @@ #include "Utils.hpp" #include "Buffer.hpp" #include "Address.hpp" -#include "C25519.hpp" #include @@ -444,49 +443,14 @@ public: return found; } - /** - * Sign this Dictionary, replacing any previous signature - * - * @param sigKey Key to use for signature in dictionary - * @param kp Key pair to sign with - */ - inline void wrapWithSignature(const char *sigKey,const C25519::Pair &kp) - { - this->erase(sigKey); - C25519::Signature sig(C25519::sign(kp,this->data(),this->sizeBytes())); - this->add(sigKey,reinterpret_cast(sig.data),ZT_C25519_SIGNATURE_LEN); - } - - /** - * Verify signature (and erase signature key) - * - * This erases this Dictionary's signature key (if present) and verifies - * the signature. The key is erased to render the Dictionary into the - * original unsigned form it was signed in for verification purposes. - * - * @param sigKey Key to use for signature in dictionary - * @param pk Public key to check against - * @return True if signature was present and valid - */ - inline bool unwrapAndVerify(const char *sigKey,const C25519::Public &pk) - { - char sig[ZT_C25519_SIGNATURE_LEN+1]; - if (this->get(sigKey,sig,sizeof(sig)) != ZT_C25519_SIGNATURE_LEN) - return false; - this->erase(sigKey); - return C25519::verify(pk,this->data(),this->sizeBytes(),sig); - } - - /** - * @return Dictionary data as a 0-terminated C-string - */ - inline const char *data() const { return _d; } - /** * @return Value of C template parameter */ inline unsigned int capacity() const { return C; } + inline const char *data() const { return _d; } + inline char *unsafeData() { return _d; } + private: char _d[C]; }; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 72dfbfd8..3988546e 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -433,21 +433,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p } break; case Packet::VERB_NETWORK_CONFIG_REQUEST: { - const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID); - const SharedPtr network(RR->node->network(nwid)); - if ((network)&&(network->controller() == peer->address())) { - trustEstablished = true; - const unsigned int chunkLen = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN); - const void *chunkData = field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,chunkLen); - unsigned int chunkIndex = 0; - unsigned int totalSize = chunkLen; - if ((ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen) < size()) { - totalSize = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen); - chunkIndex = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen + 4); - } - TRACE("%s(%s): OK(NETWORK_CONFIG_REQUEST) chunkLen==%u chunkIndex==%u totalSize==%u",source().toString().c_str(),_path->address().toString().c_str(),chunkLen,chunkIndex,totalSize); - network->handleInboundConfigChunk(inRePacketId,chunkData,chunkLen,chunkIndex,totalSize); - } + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); + if (network) + network->handleConfigChunk(*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); } break; //case Packet::VERB_ECHO: { @@ -894,20 +882,31 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons Dictionary *dconf = new Dictionary(); try { if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { - dconf->wrapWithSignature(ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE,RR->identity.privateKeyPair()); - + uint64_t configUpdateId = RR->node->prng(); + if (!configUpdateId) ++configUpdateId; const unsigned int totalSize = dconf->sizeBytes(); unsigned int chunkIndex = 0; while (chunkIndex < totalSize) { - const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PACKET_IDX_PAYLOAD + 32))); + const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); outp.append(requestPacketId); + + const unsigned int sigStart = outp.size(); outp.append(nwid); outp.append((uint16_t)chunkLen); outp.append((const void *)(dconf->data() + chunkIndex),chunkLen); + + outp.append((uint8_t)0); // no flags + outp.append((uint64_t)configUpdateId); outp.append((uint32_t)totalSize); outp.append((uint32_t)chunkIndex); + + C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + sigStart,outp.size() - sigStart)); + outp.append((uint8_t)1); + outp.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); + outp.compress(); RR->sw->send(outp,true); chunkIndex += chunkLen; @@ -977,12 +976,21 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - const uint64_t nwid = at(ZT_PACKET_IDX_PAYLOAD); - bool trustEstablished = false; - - + const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); + if (network) { + const uint64_t configUpdateId = network->handleConfigChunk(*this,ZT_PACKET_IDX_PAYLOAD); + if (configUpdateId) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((uint8_t)Packet::VERB_ECHO); + outp.append((uint64_t)packetId()); + outp.append((uint64_t)network->id()); + outp.append((uint64_t)configUpdateId); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); + } + } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,trustEstablished); + peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } diff --git a/node/Network.cpp b/node/Network.cpp index 487766a7..e24e3e16 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -569,12 +569,14 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : _lastAnnouncedMulticastGroupsUpstream(0), _mac(renv->identity.address(),nwid), _portInitialized(false), - _inboundConfigPacketId(0), _lastConfigUpdate(0), _destroyed(false), _netconfFailure(NETCONF_FAILURE_NONE), _portError(0) { + for(int i=0;i::iterator c(_inboundConfigChunks.begin());c!=_inboundConfigChunks.end();++c) - totalWeHave += (unsigned int)c->second.length(); - - if (totalWeHave == totalSize) { - TRACE("have all chunks for network config request %.16llx, assembling...",inRePacketId); - for(std::map::iterator c(_inboundConfigChunks.begin());c!=_inboundConfigChunks.end();++c) - newConfig.append(c->second); - _inboundConfigPacketId = 0; - _inboundConfigChunks.clear(); - } else if (totalWeHave > totalSize) { - _inboundConfigPacketId = 0; - _inboundConfigChunks.clear(); + const unsigned int start = ptr; + + ptr += 8; // skip network ID, which is already obviously known + const uint16_t chunkLen = chunk.at(ptr); ptr += 2; + const void *chunkData = chunk.field(ptr,chunkLen); ptr += chunkLen; + + Mutex::Lock _l(_lock); + + _IncomingConfigChunk *c = (_IncomingConfigChunk *)0; + uint64_t chunkId = 0; + uint64_t configUpdateId; + unsigned long totalLength,chunkIndex; + if (ptr < chunk.size()) { + const bool fastPropagate = ((chunk[ptr++] & 0x01) != 0); + configUpdateId = chunk.at(ptr); ptr += 8; + totalLength = chunk.at(ptr); ptr += 4; + chunkIndex = chunk.at(ptr); ptr += 4; + + if (((chunkIndex + chunkLen) > totalLength)||(totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY)) { // >= since we need room for a null at the end + TRACE("discarded chunk from %s: invalid length or length overflow",chunk.source().toString().c_str()); + return 0; + } + + if ((chunk[ptr] != 1)||(chunk.at(ptr + 1) != ZT_C25519_SIGNATURE_LEN)) { + TRACE("discarded chunk from %s: unrecognized signature type",chunk.source().toString().c_str()); + return 0; + } + const uint8_t *sig = reinterpret_cast(chunk.field(ptr + 3,ZT_C25519_SIGNATURE_LEN)); + + // We can use the signature, which is unique per chunk, to get a per-chunk ID for local deduplication use + for(unsigned int i=0;i<16;++i) + reinterpret_cast(&chunkId)[i & 7] ^= sig[i]; + + // Find existing or new slot for this update and check if this is a duplicate chunk + for(int i=0;ihaveChunks;++j) { + if (c->haveChunkIds[j] == chunkId) + return 0; + } + + break; + } else if ((!c)||(_incomingConfigChunks[i].ts < c->ts)) { + c = &(_incomingConfigChunks[i]); + } + } + + // If it's not a duplicate, check chunk signature + const Identity controllerId(RR->topology->getIdentity(controller())); + if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? + TRACE("unable to verify chunk from %s: don't have controller identity",chunk.source().toString().c_str()); + return 0; + } + if (!controllerId.verify(chunk.field(start,ptr - start),ptr - start,sig,ZT_C25519_SIGNATURE_LEN)) { + TRACE("discarded chunk from %s: signature check failed",chunk.source().toString().c_str()); + return 0; + } + + // New properly verified chunks can be flooded "virally" through the network + if (fastPropagate) { + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) { + if ((*a != chunk.source())&&(*a != controller())) { + Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CONFIG); + outp.append(reinterpret_cast(chunk.data()) + start,chunk.size() - start); + RR->sw->send(outp,true); + } + } + } + } else if (chunk.source() == controller()) { + // Legacy support for OK(NETWORK_CONFIG_REQUEST) from older controllers + chunkId = chunk.packetId(); + configUpdateId = chunkId; + totalLength = chunkLen; + chunkIndex = 0; + + if (totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY) + return 0; + + // Find oldest slot for this udpate to use buffer space + for(int i=0;its)) + c = &(_incomingConfigChunks[i]); } } else { - return; + TRACE("discarded single-chunk unsigned legacy config: this is only allowed if the sender is the controller itself"); + return 0; + } + + ++c->ts; // newer is higher, that's all we need + + if (c->updateId != configUpdateId) { + c->updateId = configUpdateId; + for(int i=0;ihaveChunkIds[i] = 0; + c->haveChunks = 0; + c->haveBytes = 0; } + if (c->haveChunks >= ZT_NETWORK_MAX_UPDATE_CHUNKS) + return false; + c->haveChunkIds[c->haveChunks++] = chunkId; + + memcpy(c->data.unsafeData() + chunkIndex,chunkData,chunkLen); + c->haveBytes += chunkLen; - if ((newConfig.length() > 0)&&(newConfig.length() < ZT_NETWORKCONFIG_DICT_CAPACITY)) { - Dictionary *dict = new Dictionary(newConfig.c_str()); - NetworkConfig *nc = new NetworkConfig(); + if (c->haveBytes == totalLength) { + c->data.unsafeData()[c->haveBytes] = (char)0; // ensure null terminated + + NetworkConfig *const nc = new NetworkConfig(); try { - Identity controllerId(RR->topology->getIdentity(this->controller())); - if (controllerId) { - if (nc->fromDictionary(*dict)) { - Mutex::Lock _l(_lock); - this->_setConfiguration(*nc,true); - } else { - TRACE("error parsing new config with length %u: deserialization of NetworkConfig failed (certificate error?)",(unsigned int)newConfig.length()); - } + if (nc->fromDictionary(c->data)) { + this->_setConfiguration(*nc,true); + return configUpdateId; } delete nc; - delete dict; } catch ( ... ) { - TRACE("error parsing new config with length %u: unexpected exception",(unsigned int)newConfig.length()); delete nc; - delete dict; - throw; } } + + return 0; } void Network::requestConfiguration() @@ -980,10 +1061,7 @@ void Network::requestConfiguration() } else { outp.append((unsigned char)0,16); } - - RR->node->expectReplyTo(_inboundConfigPacketId = outp.packetId()); - _inboundConfigChunks.clear(); - + RR->node->expectReplyTo(outp.packetId()); outp.compress(); RR->sw->send(outp,true); } @@ -1127,13 +1205,6 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c const Membership::AddCredentialResult result = m.addCredential(RR,_config,rev); if ((result == Membership::ADD_ACCEPTED_NEW)&&(rev.fastPropagate())) { - /* Fast propagation is done by using a very aggressive rumor mill - * propagation algorithm. When we see a Revocation that we haven't - * seen before we blast it to every known member. This leads to - * a huge number of redundant messages, but eventually everybody - * will get it. This helps revocation speed and also helps in cases - * where the controller is under attack. It need only get one - * revocation out and the rest is history. */ Address *a = (Address *)0; Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); diff --git a/node/Network.hpp b/node/Network.hpp index 6a1ac801..128c4668 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -44,6 +44,9 @@ #include "NetworkConfig.hpp" #include "CertificateOfMembership.hpp" +#define ZT_NETWORK_MAX_INCOMING_UPDATES 3 +#define ZT_NETWORK_MAX_UPDATE_CHUNKS ((ZT_NETWORKCONFIG_DICT_CAPACITY / 1024) + 1) + namespace ZeroTier { class RuntimeEnvironment; @@ -174,16 +177,15 @@ public: /** * Handle an inbound network config chunk * - * This is called from IncomingPacket when we receive a chunk from a network - * controller. + * This is called from IncomingPacket to handle incoming network config + * chunks via OK(NETWORK_CONFIG_REQUEST) or NETWORK_CONFIG. It verifies + * each chunk and once assembled applies the configuration. * - * @param requestId An ID for grouping chunks, e.g. in-re packet ID for OK(NETWORK_CONFIG_REQUEST) - * @param data Chunk data - * @param chunkSize Size of data[] - * @param chunkIndex Index of chunk in full config - * @param totalSize Total size of network config + * @param chunk Packet containing chunk + * @param ptr Index of chunk and related fields in packet + * @return Update ID if update was fully assembled and accepted or 0 otherwise */ - void handleInboundConfigChunk(const uint64_t requestId,const void *data,unsigned int chunkSize,unsigned int chunkIndex,unsigned int totalSize); + uint64_t handleConfigChunk(const Packet &chunk,unsigned int ptr); /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this @@ -353,19 +355,27 @@ private: const uint64_t _id; uint64_t _lastAnnouncedMulticastGroupsUpstream; MAC _mac; // local MAC address - volatile bool _portInitialized; + bool _portInitialized; std::vector< MulticastGroup > _myMulticastGroups; // multicast groups that we belong to (according to tap) Hashtable< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups that seem to be behind us and when we last saw them (if we are a bridge) Hashtable< MAC,Address > _remoteBridgeRoutes; // remote addresses where given MACs are reachable (for tracking devices behind remote bridges) - uint64_t _inboundConfigPacketId; - std::map _inboundConfigChunks; - NetworkConfig _config; - volatile uint64_t _lastConfigUpdate; + uint64_t _lastConfigUpdate; + + struct _IncomingConfigChunk + { + uint64_t ts; + uint64_t updateId; + uint64_t haveChunkIds[ZT_NETWORK_MAX_UPDATE_CHUNKS]; + unsigned long haveChunks; + unsigned long haveBytes; + Dictionary data; + }; + _IncomingConfigChunk _incomingConfigChunks[ZT_NETWORK_MAX_INCOMING_UPDATES]; - volatile bool _destroyed; + bool _destroyed; enum { NETCONF_FAILURE_NONE, @@ -373,7 +383,7 @@ private: NETCONF_FAILURE_NOT_FOUND, NETCONF_FAILURE_INIT_FAILED } _netconfFailure; - volatile int _portError; // return value from port config callback + int _portError; // return value from port config callback Hashtable _memberships; diff --git a/node/Packet.hpp b/node/Packet.hpp index b03ec327..23597f68 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -755,8 +755,26 @@ public: * <[8] 64-bit network ID> * <[2] 16-bit length of network configuration dictionary chunk> * <[...] network configuration dictionary (may be incomplete)> + * [ ... end of legacy single chunk response ... ] + * <[1] 8-bit flags> + * <[8] 64-bit config update ID (should never be 0)> * <[4] 32-bit total length of assembled dictionary> - * <[4] 32-bit index of chunk in this reply> + * <[4] 32-bit index of chunk> + * [ ... end signed portion ... ] + * <[1] 8-bit chunk signature type> + * <[2] 16-bit length of chunk signature> + * <[...] chunk signature> + * + * The chunk signature signs the entire payload of the OK response. + * Currently only one signature type is supported: ed25519 (1). + * + * Each config chunk is signed to prevent memory exhaustion or + * traffic crowding DOS attacks against config fragment assembly. + * + * If the packet is from the network controller it is permitted to end + * before the config update ID or other chunking related or signature + * fields. This is to support older controllers that don't include + * these fields and may be removed in the future. * * ERROR response payload: * <[8] 64-bit network ID> @@ -766,25 +784,30 @@ public: /** * Network configuration data push: * <[8] 64-bit network ID> - * <[8] 64-bit config update ID (token to identify this update)> - * <[1] flags> * <[2] 16-bit length of network configuration dictionary chunk> * <[...] network configuration dictionary (may be incomplete)> + * <[1] 8-bit flags> + * <[8] 64-bit config update ID (should never be 0)> * <[4] 32-bit total length of assembled dictionary> - * <[4] 32-bit index of chunk in this reply> + * <[4] 32-bit index of chunk> + * [ ... end signed portion ... ] + * <[1] 8-bit chunk signature type> + * <[2] 16-bit length of chunk signature> + * <[...] chunk signature> * * This is a direct push variant for network config updates. It otherwise - * carries the same payload as OK(NETWORK_CONFIG_REQUEST). There is an - * extra number after network ID in this version that is used in place of - * the in-re packet ID sent with OKs to group chunks together. - * - * Unlike OK(NETWORK_CONFIG_REQUEST) this can be sent by peers other than - * network controllers. In that case the certificate inside the Dictionary - * is used for verification purposes. + * carries the same payload as OK(NETWORK_CONFIG_REQUEST) and has the same + * semantics. * * Flags: - * 0x01 - Patch, not whole config - * 0x02 - Use fast P2P propagation + * 0x01 - Use fast propagation + * + * An OK should be sent if the config is successfully received and + * accepted. + * + * OK payload: + * <[8] 64-bit network ID> + * <[8] 64-bit config update ID> */ VERB_NETWORK_CONFIG = 0x0c, -- cgit v1.2.3 From cc4bacc1995d5af6b8ab66973a6d22a229367eb4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Sep 2016 12:22:25 -0700 Subject: Cleanup, and implement compression disable flag for networks. --- node/IncomingPacket.cpp | 7 ++----- node/Multicaster.cpp | 3 +++ node/Multicaster.hpp | 2 ++ node/Network.cpp | 1 - node/NetworkConfig.hpp | 10 ++++++++++ node/OutboundMulticast.cpp | 4 +++- node/OutboundMulticast.hpp | 2 ++ node/Packet.hpp | 3 +++ node/Switch.cpp | 10 +++++++--- 9 files changed, 32 insertions(+), 10 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 3988546e..0a3d58af 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -425,12 +425,12 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p RR->sa->iam(peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; - case Packet::VERB_WHOIS: { + case Packet::VERB_WHOIS: if (RR->topology->isUpstream(peer->identity())) { const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY); RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr(new Peer(RR,RR->identity,id)))); } - } break; + break; case Packet::VERB_NETWORK_CONFIG_REQUEST: { const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); @@ -438,9 +438,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p network->handleConfigChunk(*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); } break; - //case Packet::VERB_ECHO: { - //} break; - case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index fc8fa1bd..8743e8f8 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -155,6 +155,7 @@ void Multicaster::send( unsigned int limit, uint64_t now, uint64_t nwid, + bool disableCompression, const std::vector

&alwaysSendTo, const MulticastGroup &mg, const MAC &src, @@ -193,6 +194,7 @@ void Multicaster::send( RR, now, nwid, + disableCompression, limit, 1, // we'll still gather a little from peers to keep multicast list fresh src, @@ -265,6 +267,7 @@ void Multicaster::send( RR, now, nwid, + disableCompression, limit, gatherLimit, src, diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 8be3b736..5c94cd3a 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -153,6 +153,7 @@ public: * @param limit Multicast limit * @param now Current time * @param nwid Network ID + * @param disableCompression Disable packet payload compression? * @param alwaysSendTo Send to these peers first and even if not included in subscriber list * @param mg Multicast group * @param src Source Ethernet MAC address or NULL to skip in packet and compute from ZT address (non-bridged mode) @@ -164,6 +165,7 @@ public: unsigned int limit, uint64_t now, uint64_t nwid, + bool disableCompression, const std::vector
&alwaysSendTo, const MulticastGroup &mg, const MAC &src, diff --git a/node/Network.cpp b/node/Network.cpp index e24e3e16..601395d0 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -962,7 +962,6 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) if (totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY) return 0; - // Find oldest slot for this udpate to use buffer space for(int i=0;its)) c = &(_incomingConfigChunks[i]); diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 5ad86855..a548e866 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -76,6 +76,11 @@ */ #define ZT_NETWORKCONFIG_FLAG_RULES_RESULT_OF_UNSUPPORTED_MATCH 0x0000000000000008ULL +/** + * Flag: disable frame compression + */ +#define ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION 0x0000000000000010ULL + /** * Device is an active bridge */ @@ -255,6 +260,11 @@ public: */ inline bool ndpEmulation() const throw() { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0); } + /** + * @return True if frames should not be compressed + */ + inline bool disableCompression() const throw() { return ((this->flags & ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION) != 0); } + /** * @return Network type is public (no access control) */ diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 6e811581..2f6bf986 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -31,6 +31,7 @@ void OutboundMulticast::init( const RuntimeEnvironment *RR, uint64_t timestamp, uint64_t nwid, + bool disableCompression, unsigned int limit, unsigned int gatherLimit, const MAC &src, @@ -78,7 +79,8 @@ void OutboundMulticast::init( _packet.append((uint32_t)dest.adi()); _packet.append((uint16_t)etherType); _packet.append(payload,_frameLen); - _packet.compress(); + if (!disableCompression) + _packet.compress(); memcpy(_frameData,payload,_frameLen); } diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 0ded8baf..6370d0d7 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -56,6 +56,7 @@ public: * @param RR Runtime environment * @param timestamp Creation time * @param nwid Network ID + * @param disableCompression Disable compression of frame payload * @param limit Multicast limit for desired number of packets to send * @param gatherLimit Number to lazily/implicitly gather with this frame or 0 for none * @param src Source MAC address of frame or NULL to imply compute from sender ZT address @@ -69,6 +70,7 @@ public: const RuntimeEnvironment *RR, uint64_t timestamp, uint64_t nwid, + bool disableCompression, unsigned int limit, unsigned int gatherLimit, const MAC &src, diff --git a/node/Packet.hpp b/node/Packet.hpp index 23597f68..cc3d323b 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -799,6 +799,9 @@ public: * carries the same payload as OK(NETWORK_CONFIG_REQUEST) and has the same * semantics. * + * The legacy mode missing the additional chunking fields is not supported + * here. + * * Flags: * 0x01 - Use fast propagation * diff --git a/node/Switch.cpp b/node/Switch.cpp index e3d57835..6611d6b6 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -476,6 +476,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c network->config().multicastLimit, RR->node->now(), network->id(), + network->config().disableCompression(), network->config().activeBridges(), multicastGroup, (fromBridged) ? from : MAC(), @@ -501,14 +502,16 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c from.appendTo(outp); outp.append((uint16_t)etherType); outp.append(data,len); - outp.compress(); + if (!network->config().disableCompression()) + outp.compress(); send(outp,true); } else { Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME); outp.append(network->id()); outp.append((uint16_t)etherType); outp.append(data,len); - outp.compress(); + if (!network->config().disableCompression()) + outp.compress(); send(outp,true); } @@ -565,7 +568,8 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c from.appendTo(outp); outp.append((uint16_t)etherType); outp.append(data,len); - outp.compress(); + if (!network->config().disableCompression()) + outp.compress(); send(outp,true); } else { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); -- cgit v1.2.3 From 5ba7ca91c03fc3ad9ac5c360e6156b91a208fb25 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Sep 2016 12:44:44 -0700 Subject: TRACE build fix. --- node/Membership.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index b7e33936..c8fb8e4e 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -182,21 +182,21 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCapability *)0; if (have) { if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->cap.timestamp() > cap.timestamp()) ) { - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; } if (have->cap == cap) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_ACCEPTED_REDUNDANT; } } switch(cap.verify(RR)) { default: - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); + TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; case 0: - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); + TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); if (!have) have = _newCapability(cap.id()); have->lastReceived = RR->node->now(); have->cap = cap; -- cgit v1.2.3 From 9f550292fe0ebc32e61eeada9e3a69970c874724 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Sep 2016 13:49:43 -0700 Subject: Simply network auth logic and always sent error on auth failure even for unknown networks to prevent forensics. --- node/IncomingPacket.cpp | 61 ++++++++++++++++++++++++++++++------------------- node/IncomingPacket.hpp | 2 ++ node/Network.cpp | 21 +++-------------- node/Network.hpp | 15 +----------- node/Node.hpp | 8 ++++--- node/Peer.cpp | 1 + node/Peer.hpp | 17 ++++++++++++-- 7 files changed, 64 insertions(+), 61 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 0a3d58af..f54752d1 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -169,7 +169,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // Peers can send this in response to frames if they do not have a recent enough COM from us const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); const uint64_t now = RR->node->now(); - if ( (network) && (network->config().com) && (peer->rateGateComRequest(now)) ) + if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) network->pushCredentialsNow(peer->address(),now); } break; @@ -184,7 +184,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // Members of networks can use this error to indicate that they no longer // want to receive multicasts on a given channel. const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->gate(peer,verb(),packetId()))) { + if ((network)&&(network->gate(peer))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); RR->mc->remove(network->id(),mg,peer->address()); @@ -375,7 +375,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); - bool trustEstablished = false; if (!RR->node->expectingReplyTo(inRePacketId)) { TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId()); @@ -441,8 +440,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_MULTICAST_GATHER: { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); - if ((network)&&(network->gateMulticastGatherReply(peer,verb(),packetId()))) { - trustEstablished = true; + if (network) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); @@ -468,15 +466,12 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p network->addCredential(com); } - if (network->gateMulticastGatherReply(peer,verb(),packetId())) { - trustEstablished = true; - if ((flags & 0x02) != 0) { - // OK(MULTICAST_FRAME) includes implicit gather results - offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; - unsigned int totalKnown = at(offset); offset += 4; - unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); - } + if ((flags & 0x02) != 0) { + // OK(MULTICAST_FRAME) includes implicit gather results + offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; + unsigned int totalKnown = at(offset); offset += 4; + unsigned int count = at(offset); offset += 2; + RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); } } } break; @@ -484,7 +479,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,trustEstablished); + peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } @@ -581,9 +576,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr const SharedPtr network(RR->node->network(nwid)); bool trustEstablished = false; if (network) { - if (!network->gate(peer,verb(),packetId())) { - TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - } else { + if (network->gate(peer)) { trustEstablished = true; if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); @@ -593,9 +586,13 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); } + } else { + TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + _sendErrorNeedCredentials(RR,peer,nwid); } } else { TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); + _sendErrorNeedCredentials(RR,peer,nwid); } peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { @@ -620,8 +617,9 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

addCredential(com); } - if (!network->gate(peer,verb(),packetId())) { + if (!network->gate(peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); + _sendErrorNeedCredentials(RR,peer,nwid); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -681,6 +679,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); + _sendErrorNeedCredentials(RR,peer,nwid); peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { @@ -737,7 +736,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared if (!auth) { if ((!network)||(network->id() != nwid)) network = RR->node->network(nwid); - const bool authOnNet = ((network)&&(network->gate(peer,verb(),packetId()))); + const bool authOnNet = ((network)&&(network->gate(peer))); trustEstablished |= authOnNet; if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { auth = true; @@ -986,7 +985,6 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared _path->send(RR,outp.data(),outp.size(),RR->node->now()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -1020,7 +1018,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar } } - const bool trustEstablished = ((network)&&(network->gate(peer,verb(),packetId()))); + const bool trustEstablished = ((network)&&(network->gate(peer))); if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); @@ -1067,8 +1065,9 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share network->addCredential(com); } - if (!network->gate(peer,verb(),packetId())) { + if (!network->gate(peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + _sendErrorNeedCredentials(RR,peer,nwid); peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -1143,6 +1142,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); } else { + _sendErrorNeedCredentials(RR,peer,nwid); peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { @@ -1288,7 +1288,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } - if (network->gate(peer,verb(),packetId())) + if (network->gate(peer)) reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); @@ -1479,6 +1479,19 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const return true; } +void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid) +{ + if (peer->rateGateOutgoingComRequest(RR->node->now())) { + Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((uint8_t)verb()); + outp.append(packetId()); + outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); + outp.append(nwid); + outp.armor(peer->key(),true); + _path->send(RR,outp.data(),outp.size(),RR->node->now()); + } +} + void IncomingPacket::computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16]) { unsigned char salsabuf[131072]; // 131072 == protocol constant, size of memory buffer for this proof of work function diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 86c2b5e7..c3632216 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -154,6 +154,8 @@ private: bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr &peer); + void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid); + uint64_t _receiveTime; SharedPtr _path; }; diff --git a/node/Network.cpp b/node/Network.cpp index 601395d0..52abbcf9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -50,6 +50,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_ACTION_DROP: return "ACTION_DROP"; case ZT_NETWORK_RULE_ACTION_ACCEPT: return "ACTION_ACCEPT"; case ZT_NETWORK_RULE_ACTION_TEE: return "ACTION_TEE"; + case ZT_NETWORK_RULE_ACTION_WATCH: return "ACTION_WATCH"; case ZT_NETWORK_RULE_ACTION_REDIRECT: return "ACTION_REDIRECT"; case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: return "ACTION_DEBUG_LOG"; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: return "MATCH_SOURCE_ZEROTIER_ADDRESS"; @@ -882,7 +883,7 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) const unsigned int start = ptr; ptr += 8; // skip network ID, which is already obviously known - const uint16_t chunkLen = chunk.at(ptr); ptr += 2; + const unsigned int chunkLen = chunk.at(ptr); ptr += 2; const void *chunkData = chunk.field(ptr,chunkLen); ptr += chunkLen; Mutex::Lock _l(_lock); @@ -975,8 +976,6 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) if (c->updateId != configUpdateId) { c->updateId = configUpdateId; - for(int i=0;ihaveChunkIds[i] = 0; c->haveChunks = 0; c->haveBytes = 0; } @@ -1065,7 +1064,7 @@ void Network::requestConfiguration() RR->sw->send(outp,true); } -bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) +bool Network::gate(const SharedPtr &peer) { const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); @@ -1081,15 +1080,6 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin m->likingMulticasts(now); } return true; - } else { - if (peer->rateGateRequestCredentials(now)) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((uint8_t)verb); - outp.append(packetId); - outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); - outp.append(_id); - RR->sw->send(outp,true); - } } } } catch ( ... ) { @@ -1098,11 +1088,6 @@ bool Network::gate(const SharedPtr &peer,const Packet::Verb verb,const uin return false; } -bool Network::gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId) -{ - return ( (peer->address() == controller()) || RR->topology->isUpstream(peer->identity()) || gate(peer,verb,packetId) || _config.isAnchor(peer->address()) ); -} - void Network::clean() { const uint64_t now = RR->node->now(); diff --git a/node/Network.hpp b/node/Network.hpp index 128c4668..527d3048 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -212,21 +212,8 @@ public: /** * Determine whether this peer is permitted to communicate on this network - * - * This also performs certain periodic actions such as pushing renewed - * credentials to peers, so like the filters it is not side-effect-free. - * - * @param peer Peer to check - * @param verb Packet verb - * @param packetId Packet ID - * @return True if peer is allowed to communicate on this network - */ - bool gate(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); - - /** - * Check whether this peer is allowed to provide multicast info for this network */ - bool gateMulticastGatherReply(const SharedPtr &peer,const Packet::Verb verb,const uint64_t packetId); + bool gate(const SharedPtr &peer); /** * Do periodic cleanup and housekeeping tasks diff --git a/node/Node.hpp b/node/Node.hpp index 11462531..ddc52651 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -267,17 +267,19 @@ public: } /** - * Check whether a given packet ID is something we are expecting a reply to + * Check whether a given packet ID is something we are expecting a reply to (and erase from list) * * @param packetId Packet ID to check * @return True if we're expecting a reply */ - inline bool expectingReplyTo(const uint64_t packetId) const + inline bool expectingReplyTo(const uint64_t packetId) { const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) { - if (_expectingRepliesTo[bucket][i] == packetId) + if (_expectingRepliesTo[bucket][i] == packetId) { + _expectingRepliesTo[bucket][i] = 0; return true; + } } return false; } diff --git a/node/Peer.cpp b/node/Peer.cpp index d742964a..87882dad 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -50,6 +50,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastWhoisRequestReceived(0), _lastEchoRequestReceived(0), _lastComRequestReceived(0), + _lastComRequestSent(0), _lastCredentialsReceived(0), _lastTrustEstablishedPacketReceived(0), RR(renv), diff --git a/node/Peer.hpp b/node/Peer.hpp index c5ef43ed..d0589ccf 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -395,9 +395,9 @@ public: } /** - * Rate gate requests for network COM + * Rate gate incoming requests for network COM */ - inline bool rateGateComRequest(const uint64_t now) + inline bool rateGateIncomingComRequest(const uint64_t now) { if ((now - _lastComRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastComRequestReceived = now; @@ -406,6 +406,18 @@ public: return false; } + /** + * Rate gate outgoing requests for network COM + */ + inline bool rateGateOutgoingComRequest(const uint64_t now) + { + if ((now - _lastComRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { + _lastComRequestSent = now; + return true; + } + return false; + } + /** * Find a common set of addresses by which two peers can link, if any * @@ -465,6 +477,7 @@ private: uint64_t _lastWhoisRequestReceived; uint64_t _lastEchoRequestReceived; uint64_t _lastComRequestReceived; + uint64_t _lastComRequestSent; uint64_t _lastCredentialsReceived; uint64_t _lastTrustEstablishedPacketReceived; const RuntimeEnvironment *RR; -- cgit v1.2.3 From 0b44919ba23021231dd561f530c5d30836846735 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Sep 2016 16:33:37 -0700 Subject: Clusters can send multiple OKs so we must allow this. --- node/IncomingPacket.cpp | 5 +++-- node/Node.hpp | 8 +++----- 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index f54752d1..b77ead4c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1481,14 +1481,15 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid) { - if (peer->rateGateOutgoingComRequest(RR->node->now())) { + const uint64_t now = RR->node->now(); + if (peer->rateGateOutgoingComRequest(now)) { Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)verb()); outp.append(packetId()); outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); outp.append(nwid); outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,outp.data(),outp.size(),now); } } diff --git a/node/Node.hpp b/node/Node.hpp index ddc52651..11462531 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -267,19 +267,17 @@ public: } /** - * Check whether a given packet ID is something we are expecting a reply to (and erase from list) + * Check whether a given packet ID is something we are expecting a reply to * * @param packetId Packet ID to check * @return True if we're expecting a reply */ - inline bool expectingReplyTo(const uint64_t packetId) + inline bool expectingReplyTo(const uint64_t packetId) const { const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) { - if (_expectingRepliesTo[bucket][i] == packetId) { - _expectingRepliesTo[bucket][i] = 0; + if (_expectingRepliesTo[bucket][i] == packetId) return true; - } } return false; } -- cgit v1.2.3 From 5ee1ccd65987353dd461f8ea27203da63c0b2cd8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Sep 2016 16:41:08 -0700 Subject: Send need credential error on more cases. --- node/IncomingPacket.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b77ead4c..dd95f8c8 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -737,6 +737,8 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared if ((!network)||(network->id() != nwid)) network = RR->node->network(nwid); const bool authOnNet = ((network)&&(network->gate(peer))); + if (!authOnNet) + _sendErrorNeedCredentials(RR,peer,nwid); trustEstablished |= authOnNet; if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { auth = true; @@ -1019,6 +1021,8 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar } const bool trustEstablished = ((network)&&(network->gate(peer))); + if (!trustEstablished) + _sendErrorNeedCredentials(RR,peer,nwid); if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); -- cgit v1.2.3 From 7e90ab3534e414ef9dde93ec5a74ee2717092fb8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 28 Sep 2016 11:06:44 -0700 Subject: TRACE verbosity increase on exceptions in NETWORK_CREDENTIALS. --- node/IncomingPacket.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index dd95f8c8..e0fa3bf1 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -852,6 +852,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); + } catch (std::exception &exc) { + TRACE("dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } -- cgit v1.2.3 From e1fbf7b34c0e9db6c6845238bc553fc470857fef Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 28 Sep 2016 12:21:08 -0700 Subject: Check multicast limit on send after NDP emulation code. --- node/Switch.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 6611d6b6..75898d21 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -349,11 +349,6 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } if (to.isMulticast()) { - if (network->config().multicastLimit == 0) { - TRACE("%.16llx: dropped multicast: not allowed on network",network->id()); - return; - } - MulticastGroup multicastGroup(to,0); if (to.isBroadcast()) { @@ -457,6 +452,12 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } // else no NDP emulation } + // Check this after NDP emulation, since that has to be allowed in exactly this case + if (network->config().multicastLimit == 0) { + TRACE("%.16llx: dropped multicast: not allowed on network",network->id()); + return; + } + /* Learn multicast groups for bridged-in hosts. * Note that some OSes, most notably Linux, do this for you by learning * multicast addresses on bridge interfaces and subscribing each slave. -- cgit v1.2.3 From 01129d02b3863f92c0f94b29e9600153b743f967 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 28 Sep 2016 13:45:25 -0700 Subject: hashCode() for InetAddress --- node/InetAddress.hpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'node') diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 2cd4dbd3..6f070fbf 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -414,6 +414,25 @@ struct InetAddress : public sockaddr_storage return false; } + inline unsigned long hashCode() const + { + if (ss_family == AF_INET) { + return ((unsigned long)reinterpret_cast(this)->sin_addr.s_addr + (unsigned long)reinterpret_cast(this)->sin_port); + } else if (ss_family == AF_INET6) { + unsigned long tmp = reinterpret_cast(this)->sin6_port; + const uint8_t *a = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); + for(long i=0;i<16;++i) + reinterpret_cast(&tmp)[i % sizeof(tmp)] ^= a[i]; + return tmp; + } else { + unsigned long tmp = reinterpret_cast(this)->sin6_port; + const uint8_t *a = reinterpret_cast(this); + for(long i=0;i(&tmp)[i % sizeof(tmp)] ^= a[i]; + return tmp; + } + } + /** * Set to null/zero */ -- cgit v1.2.3 From 4fe9a4fe8376237ffba684a3e0be2edb14527fe1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 28 Sep 2016 16:13:59 -0700 Subject: Fix memory leak. --- node/Network.cpp | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 52abbcf9..ac679e46 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -993,12 +993,11 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) try { if (nc->fromDictionary(c->data)) { this->_setConfiguration(*nc,true); + delete nc; return configUpdateId; } - delete nc; - } catch ( ... ) { - delete nc; - } + } catch ( ... ) {} + delete nc; } return 0; @@ -1025,25 +1024,31 @@ void Network::requestConfiguration() if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { - NetworkConfig nconf; - switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,nconf)) { - case NetworkController::NETCONF_QUERY_OK: { - Mutex::Lock _l(_lock); - this->_setConfiguration(nconf,true); - } return; - case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: - this->setNotFound(); - return; - case NetworkController::NETCONF_QUERY_ACCESS_DENIED: - this->setAccessDenied(); - return; - default: - return; + NetworkConfig *nconf = new NetworkConfig(); + try { + switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,*nconf)) { + case NetworkController::NETCONF_QUERY_OK: { + Mutex::Lock _l(_lock); + this->_setConfiguration(*nconf,true); + } break; + case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: + this->setNotFound(); + break; + case NetworkController::NETCONF_QUERY_ACCESS_DENIED: + this->setAccessDenied(); + break; + default: + this->setNotFound(); + break; + } + } catch ( ... ) { + this->setNotFound(); } + delete nconf; } else { this->setNotFound(); - return; } + return; } TRACE("requesting netconf for network %.16llx from controller %s",(unsigned long long)_id,ctrl.toString().c_str()); -- cgit v1.2.3 From 9eaa3756f8dc711b6b828c8336ed34185b4b5834 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Sep 2016 12:22:54 -0700 Subject: Fix deadlock-causing regression in Network. --- node/Network.cpp | 230 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 124 insertions(+), 106 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index ac679e46..d38a3fdd 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -886,118 +886,130 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) const unsigned int chunkLen = chunk.at(ptr); ptr += 2; const void *chunkData = chunk.field(ptr,chunkLen); ptr += chunkLen; - Mutex::Lock _l(_lock); - - _IncomingConfigChunk *c = (_IncomingConfigChunk *)0; - uint64_t chunkId = 0; + NetworkConfig *nc = (NetworkConfig *)0; uint64_t configUpdateId; - unsigned long totalLength,chunkIndex; - if (ptr < chunk.size()) { - const bool fastPropagate = ((chunk[ptr++] & 0x01) != 0); - configUpdateId = chunk.at(ptr); ptr += 8; - totalLength = chunk.at(ptr); ptr += 4; - chunkIndex = chunk.at(ptr); ptr += 4; - - if (((chunkIndex + chunkLen) > totalLength)||(totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY)) { // >= since we need room for a null at the end - TRACE("discarded chunk from %s: invalid length or length overflow",chunk.source().toString().c_str()); - return 0; - } + { + Mutex::Lock _l(_lock); + + _IncomingConfigChunk *c = (_IncomingConfigChunk *)0; + uint64_t chunkId = 0; + unsigned long totalLength,chunkIndex; + if (ptr < chunk.size()) { + const bool fastPropagate = ((chunk[ptr++] & 0x01) != 0); + configUpdateId = chunk.at(ptr); ptr += 8; + totalLength = chunk.at(ptr); ptr += 4; + chunkIndex = chunk.at(ptr); ptr += 4; + + if (((chunkIndex + chunkLen) > totalLength)||(totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY)) { // >= since we need room for a null at the end + TRACE("discarded chunk from %s: invalid length or length overflow",chunk.source().toString().c_str()); + return 0; + } - if ((chunk[ptr] != 1)||(chunk.at(ptr + 1) != ZT_C25519_SIGNATURE_LEN)) { - TRACE("discarded chunk from %s: unrecognized signature type",chunk.source().toString().c_str()); - return 0; - } - const uint8_t *sig = reinterpret_cast(chunk.field(ptr + 3,ZT_C25519_SIGNATURE_LEN)); + if ((chunk[ptr] != 1)||(chunk.at(ptr + 1) != ZT_C25519_SIGNATURE_LEN)) { + TRACE("discarded chunk from %s: unrecognized signature type",chunk.source().toString().c_str()); + return 0; + } + const uint8_t *sig = reinterpret_cast(chunk.field(ptr + 3,ZT_C25519_SIGNATURE_LEN)); - // We can use the signature, which is unique per chunk, to get a per-chunk ID for local deduplication use - for(unsigned int i=0;i<16;++i) - reinterpret_cast(&chunkId)[i & 7] ^= sig[i]; + // We can use the signature, which is unique per chunk, to get a per-chunk ID for local deduplication use + for(unsigned int i=0;i<16;++i) + reinterpret_cast(&chunkId)[i & 7] ^= sig[i]; - // Find existing or new slot for this update and check if this is a duplicate chunk - for(int i=0;ihaveChunks;++j) { - if (c->haveChunkIds[j] == chunkId) - return 0; - } + for(unsigned long j=0;jhaveChunks;++j) { + if (c->haveChunkIds[j] == chunkId) + return 0; + } - break; - } else if ((!c)||(_incomingConfigChunks[i].ts < c->ts)) { - c = &(_incomingConfigChunks[i]); + break; + } else if ((!c)||(_incomingConfigChunks[i].ts < c->ts)) { + c = &(_incomingConfigChunks[i]); + } } - } - // If it's not a duplicate, check chunk signature - const Identity controllerId(RR->topology->getIdentity(controller())); - if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? - TRACE("unable to verify chunk from %s: don't have controller identity",chunk.source().toString().c_str()); - return 0; - } - if (!controllerId.verify(chunk.field(start,ptr - start),ptr - start,sig,ZT_C25519_SIGNATURE_LEN)) { - TRACE("discarded chunk from %s: signature check failed",chunk.source().toString().c_str()); - return 0; - } + // If it's not a duplicate, check chunk signature + const Identity controllerId(RR->topology->getIdentity(controller())); + if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? + TRACE("unable to verify chunk from %s: don't have controller identity",chunk.source().toString().c_str()); + return 0; + } + if (!controllerId.verify(chunk.field(start,ptr - start),ptr - start,sig,ZT_C25519_SIGNATURE_LEN)) { + TRACE("discarded chunk from %s: signature check failed",chunk.source().toString().c_str()); + return 0; + } - // New properly verified chunks can be flooded "virally" through the network - if (fastPropagate) { - Address *a = (Address *)0; - Membership *m = (Membership *)0; - Hashtable::Iterator i(_memberships); - while (i.next(a,m)) { - if ((*a != chunk.source())&&(*a != controller())) { - Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CONFIG); - outp.append(reinterpret_cast(chunk.data()) + start,chunk.size() - start); - RR->sw->send(outp,true); + // New properly verified chunks can be flooded "virally" through the network + if (fastPropagate) { + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) { + if ((*a != chunk.source())&&(*a != controller())) { + Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CONFIG); + outp.append(reinterpret_cast(chunk.data()) + start,chunk.size() - start); + RR->sw->send(outp,true); + } } } - } - } else if (chunk.source() == controller()) { - // Legacy support for OK(NETWORK_CONFIG_REQUEST) from older controllers - chunkId = chunk.packetId(); - configUpdateId = chunkId; - totalLength = chunkLen; - chunkIndex = 0; - - if (totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY) + } else if (chunk.source() == controller()) { + // Legacy support for OK(NETWORK_CONFIG_REQUEST) from older controllers + chunkId = chunk.packetId(); + configUpdateId = chunkId; + totalLength = chunkLen; + chunkIndex = 0; + + if (totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY) + return 0; + + for(int i=0;its)) + c = &(_incomingConfigChunks[i]); + } + } else { + TRACE("discarded single-chunk unsigned legacy config: this is only allowed if the sender is the controller itself"); return 0; - - for(int i=0;its)) - c = &(_incomingConfigChunks[i]); } - } else { - TRACE("discarded single-chunk unsigned legacy config: this is only allowed if the sender is the controller itself"); - return 0; - } - ++c->ts; // newer is higher, that's all we need + ++c->ts; // newer is higher, that's all we need - if (c->updateId != configUpdateId) { - c->updateId = configUpdateId; - c->haveChunks = 0; - c->haveBytes = 0; - } - if (c->haveChunks >= ZT_NETWORK_MAX_UPDATE_CHUNKS) - return false; - c->haveChunkIds[c->haveChunks++] = chunkId; + if (c->updateId != configUpdateId) { + c->updateId = configUpdateId; + c->haveChunks = 0; + c->haveBytes = 0; + } + if (c->haveChunks >= ZT_NETWORK_MAX_UPDATE_CHUNKS) + return false; + c->haveChunkIds[c->haveChunks++] = chunkId; - memcpy(c->data.unsafeData() + chunkIndex,chunkData,chunkLen); - c->haveBytes += chunkLen; + memcpy(c->data.unsafeData() + chunkIndex,chunkData,chunkLen); + c->haveBytes += chunkLen; - if (c->haveBytes == totalLength) { - c->data.unsafeData()[c->haveBytes] = (char)0; // ensure null terminated + if (c->haveBytes == totalLength) { + c->data.unsafeData()[c->haveBytes] = (char)0; // ensure null terminated - NetworkConfig *const nc = new NetworkConfig(); - try { - if (nc->fromDictionary(c->data)) { - this->_setConfiguration(*nc,true); + nc = new NetworkConfig(); + try { + if (!nc->fromDictionary(c->data)) { + delete nc; + nc = (NetworkConfig *)0; + } + } catch ( ... ) { delete nc; - return configUpdateId; + nc = (NetworkConfig *)0; } - } catch ( ... ) {} + } + } + + if (nc) { + this->_setConfiguration(*nc,true); delete nc; + return configUpdateId; + } else { + return 0; } return 0; @@ -1027,10 +1039,9 @@ void Network::requestConfiguration() NetworkConfig *nconf = new NetworkConfig(); try { switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,*nconf)) { - case NetworkController::NETCONF_QUERY_OK: { - Mutex::Lock _l(_lock); + case NetworkController::NETCONF_QUERY_OK: this->_setConfiguration(*nconf,true); - } break; + break; case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: this->setNotFound(); break; @@ -1238,7 +1249,7 @@ ZT_VirtualNetworkStatus Network::_status() const int Network::_setConfiguration(const NetworkConfig &nconf,bool saveToDisk) { - // assumes _lock is locked + // _lock is NOT locked when this is called try { if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) return 0; @@ -1246,20 +1257,27 @@ int Network::_setConfiguration(const NetworkConfig &nconf,bool saveToDisk) return 1; // OK config, but duplicate of what we already have ZT_VirtualNetworkConfig ctmp; - _config = nconf; - _lastConfigUpdate = RR->node->now(); - _netconfFailure = NETCONF_FAILURE_NONE; - _externalConfig(&ctmp); - const bool oldPortInitialized = _portInitialized; - _portInitialized = true; + bool oldPortInitialized; + { + Mutex::Lock _l(_lock); + _config = nconf; + _lastConfigUpdate = RR->node->now(); + _netconfFailure = NETCONF_FAILURE_NONE; + oldPortInitialized = _portInitialized; + _portInitialized = true; + _externalConfig(&ctmp); + } _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { - char n[64]; - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - Dictionary d; - if (nconf.toDictionary(d,false)) - RR->node->dataStorePut(n,(const void *)d.data(),d.sizeBytes(),true); + Dictionary *d = new Dictionary(); + try { + char n[64]; + Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); + if (nconf.toDictionary(*d,false)) + RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); + } catch ( ... ) {} + delete d; } return 2; // OK and configuration has changed -- cgit v1.2.3 From 988049f39bf7731c0189544b316850b81b004de3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Sep 2016 14:07:00 -0700 Subject: Add new rule to rules engine: random match. --- controller/EmbeddedNetworkController.cpp | 8 ++++++++ include/ZeroTierOne.h | 18 ++++++++++++++---- node/Capability.hpp | 7 +++++++ node/Network.cpp | 4 ++++ 4 files changed, 33 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c60f7322..d656bad3 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -260,6 +260,11 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["start"] = (unsigned int)rule.v.frameSize[0]; r["end"] = (unsigned int)rule.v.frameSize[1]; break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + r["type"] = "MATCH_RANDOM"; + r["not"] = ((rule.t & 0x80) != 0); + r["probability"] = (unsigned long)rule.v.randomProbability; + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: r["type"] = "MATCH_TAGS_DIFFERENCE"; r["not"] = ((rule.t & 0x80) != 0); @@ -441,6 +446,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.frameSize[0] = (uint16_t)(_jI(r["start"],0ULL) & 0xffffULL); rule.v.frameSize[1] = (uint16_t)(_jI(r["end"],(uint64_t)rule.v.frameSize[0]) & 0xffffULL); return true; + } else if (t == "MATCH_RANDOM") { + rule.t |= ZT_NETWORK_RULE_MATCH_RANDOM; + rule.v.randomProbability = (uint32_t)(_jI(r["probability"],0ULL) & 0xffffffffULL); } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index c66b9079..ee03c3b1 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -633,25 +633,30 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 50, + /** + * Random match with selectable probability + */ + ZT_NETWORK_RULE_MATCH_RANDOM = 51, + /** * Match if local and remote tags differ by no more than value, use 0 to check for equality */ - ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 51, + ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 52, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 52, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 53, /** * Match if local and remote tags ANDed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 53, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 54, /** * Match if local and remote tags XORed together equal value. */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 54, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 55, /** * Maximum ID allowed for a MATCH entry in the rules table @@ -720,6 +725,11 @@ typedef struct */ uint64_t zt; + /** + * 0 = never, UINT32_MAX = always + */ + uint32_t randomProbability; + /** * 48-bit Ethernet MAC address in big-endian order */ diff --git a/node/Capability.hpp b/node/Capability.hpp index 2cf54b5c..e808ad40 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -249,6 +249,10 @@ public: b.append((uint16_t)rules[i].v.frameSize[0]); b.append((uint16_t)rules[i].v.frameSize[1]); break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + b.append((uint8_t)4); + b.append((uint32_t)rules[i].v.randomProbability); + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: @@ -331,6 +335,9 @@ public: rules[ruleCount].v.frameSize[0] = b.template at(p); rules[ruleCount].v.frameSize[1] = b.template at(p + 2); break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + rules[ruleCount].v.randomProbability = b.template at(p); + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: diff --git a/node/Network.cpp b/node/Network.cpp index d38a3fdd..fe899dcc 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -504,6 +504,10 @@ static _doZtFilterResult _doZtFilter( thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + thisRuleMatches = (uint8_t)((uint32_t)(RR->node->prng() & 0xffffffffULL) <= rules[rn].v.randomProbability); + FILTER_TRACE("%u %s %c -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)thisRuleMatches); + break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: -- cgit v1.2.3 From d5f4d381d01a4428558627feb42cea70fa7a90e2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 5 Oct 2016 10:12:06 -0700 Subject: Go ahead and loop back packets whose destination is self. Some OSes require this since they aactually follow the full network path even for local IPs. --- node/Switch.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 75898d21..82b13483 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -334,18 +334,13 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c if (!network->hasConfig()) return; - // Sanity check -- bridge loop? OS problem? - if (to == network->mac()) - return; - // Check if this packet is from someone other than the tap -- i.e. bridged in - bool fromBridged = false; - if (from != network->mac()) { + bool fromBridged; + if ((fromBridged = (from != network->mac()))) { if (!network->config().permitsBridging(RR->identity.address())) { TRACE("%.16llx: %s -> %s %s not forwarded, bridging disabled or this peer not a bridge",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } - fromBridged = true; } if (to.isMulticast()) { @@ -484,6 +479,9 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c etherType, data, len); + } else if (to == network->mac()) { + // Destination is this node, so just reinject it + RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,vlanId,data,len); } else if (to[0] == MAC::firstOctetForNetwork(network->id())) { // Destination is another ZeroTier peer on the same network -- cgit v1.2.3 From adeb7e7da0e5d1e267c272a4f1d1c9b731e291d9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 5 Oct 2016 12:54:46 -0700 Subject: Make capability flags match more user-friendly and appropriate since "match any flag" is generally what we want. --- controller/EmbeddedNetworkController.cpp | 19 ++++--------------- include/ZeroTierOne.h | 2 +- node/Capability.hpp | 8 +++----- node/Network.cpp | 2 +- 4 files changed, 9 insertions(+), 22 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d656bad3..7fa66224 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -249,10 +249,8 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); r["mask"] = tmp; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); - r["value"] = tmp; break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: r["type"] = "MATCH_FRAME_SIZE_RANGE"; @@ -423,21 +421,12 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_CHARACTERISTICS") { rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; if (r.count("mask")) { - auto v = r["mask"]; + json &v = r["mask"]; if (v.is_number()) { - rule.v.characteristics[0] = v; + rule.v.characteristics = v; } else { std::string tmp = v; - rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); - } - } - if (r.count("value")) { - auto v = r["value"]; - if (v.is_number()) { - rule.v.characteristics[1] = v; - } else { - std::string tmp = v; - rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); + rule.v.characteristics = Utils::hexStrToU64(tmp.c_str()); } } return true; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index ee03c3b1..e231ae62 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -713,7 +713,7 @@ typedef struct /** * Packet characteristic flags being matched */ - uint64_t characteristics[2]; + uint64_t characteristics; /** * IP port range -- start-end inclusive -- host byte order diff --git a/node/Capability.hpp b/node/Capability.hpp index e808ad40..f757639d 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -240,9 +240,8 @@ public: b.append((uint16_t)rules[i].v.port[1]); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - b.append((uint8_t)16); - b.append((uint64_t)rules[i].v.characteristics[0]); - b.append((uint64_t)rules[i].v.characteristics[1]); + b.append((uint8_t)8); + b.append((uint64_t)rules[i].v.characteristics); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: b.append((uint8_t)4); @@ -328,8 +327,7 @@ public: rules[ruleCount].v.port[1] = b.template at(p + 2); break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - rules[ruleCount].v.characteristics[0] = b.template at(p); - rules[ruleCount].v.characteristics[1] = b.template at(p + 8); + rules[ruleCount].v.characteristics = b.template at(p); break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: rules[ruleCount].v.frameSize[0] = b.template at(p); diff --git a/node/Network.cpp b/node/Network.cpp index fe899dcc..8b9f6e3d 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -497,7 +497,7 @@ static _doZtFilterResult _doZtFilter( } } } - thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics[0]) == rules[rn].v.characteristics[1]); + thisRuleMatches = (uint8_t)((cf | rules[rn].v.characteristics) != 0); FILTER_TRACE("%u %s %c (%.16llx & %.16llx)==%.16llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: -- cgit v1.2.3 From 45c4ccb15362e17ec7030287a314df19a830f0f3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 5 Oct 2016 16:38:42 -0700 Subject: Add a tags both equal match. --- controller/EmbeddedNetworkController.cpp | 11 +++++++++++ include/ZeroTierOne.h | 5 +++++ node/Capability.hpp | 1 + node/Network.cpp | 6 +++++- 4 files changed, 22 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7fa66224..aa95ac64 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -287,6 +287,12 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; break; + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + r["type"] = "MATCH_TAGS_EQUAL"; + r["not"] = ((rule.t & 0x80) != 0); + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; } return r; } @@ -458,6 +464,11 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; + } else if (t == "MATCH_TAGS_EQUAL") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_EQUAL; + rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); + return true; } return false; } diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e231ae62..8d7b0cd4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -658,6 +658,11 @@ enum ZT_VirtualNetworkRuleType */ ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 55, + /** + * Match if local and remote tags both equal a value + */ + ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 56, + /** * Maximum ID allowed for a MATCH entry in the rules table */ diff --git a/node/Capability.hpp b/node/Capability.hpp index f757639d..99980ce7 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -340,6 +340,7 @@ public: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: rules[ruleCount].v.tag.id = b.template at(p); rules[ruleCount].v.tag.value = b.template at(p + 4); break; diff --git a/node/Network.cpp b/node/Network.cpp index 8b9f6e3d..00c201ba 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -511,7 +511,8 @@ static _doZtFilterResult _doZtFilter( case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: { + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: { const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); @@ -531,6 +532,9 @@ static _doZtFilterResult _doZtFilter( } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { thisRuleMatches = (uint8_t)((ltv ^ rtv) == rules[rn].v.tag.value); FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_EQUAL) { + thisRuleMatches = (uint8_t)((ltv == rules[rn].v.tag.value)&&(rtv == rules[rn].v.tag.value)); + FILTER_TRACE("%u %s %c TAG %u local:%.8x and remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } -- cgit v1.2.3 From 6a50291aa20cfca82f7648b3d6ff529cbe287c51 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Fri, 7 Oct 2016 14:29:06 -0700 Subject: Fix the case for InetAddress::containsAddress for IPv6 route of :: --- node/InetAddress.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 12446909..7d22eeae 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -236,8 +236,14 @@ InetAddress InetAddress::netmask() const case AF_INET6: { uint64_t nm[2]; const unsigned int bits = netmaskBits(); - nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); - nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); + if(bits) { + nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); + nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); + } + else { + nm[0] = 0; + nm[1] = 0; + } memcpy(reinterpret_cast(&r)->sin6_addr.s6_addr,nm,16); } break; } -- cgit v1.2.3 From e53f63ca8700a526b15c2e7d05076d685734bcf6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 11 Oct 2016 12:00:16 -0700 Subject: Broke down and added an OR to the rules engine. It is now possible to have a series of MATCHes that are ORed. --- controller/EmbeddedNetworkController.cpp | 282 +++++++++++++++---------------- include/ZeroTierOne.h | 163 ++++-------------- node/Capability.hpp | 18 +- node/Network.cpp | 22 +-- 4 files changed, 193 insertions(+), 292 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index aa95ac64..7559cf83 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -127,7 +127,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) { char tmp[128]; json r = json::object(); - switch((rule.t) & 0x7f) { + const ZT_VirtualNetworkRuleType rt = (ZT_VirtualNetworkRuleType)(rule.t & 0x3f); + + switch(rt) { case ZT_NETWORK_RULE_ACTION_DROP: r["type"] = "ACTION_DROP"; break; @@ -154,146 +156,136 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: r["type"] = "ACTION_DEBUG_LOG"; break; - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; - r["not"] = ((rule.t & 0x80) != 0); - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; - r["not"] = ((rule.t & 0x80) != 0); - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - r["type"] = "MATCH_VLAN_ID"; - r["not"] = ((rule.t & 0x80) != 0); - r["vlanId"] = (unsigned int)rule.v.vlanId; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - r["type"] = "MATCH_VLAN_PCP"; - r["not"] = ((rule.t & 0x80) != 0); - r["vlanPcp"] = (unsigned int)rule.v.vlanPcp; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - r["type"] = "MATCH_VLAN_DEI"; - r["not"] = ((rule.t & 0x80) != 0); - r["vlanDei"] = (unsigned int)rule.v.vlanDei; - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - r["type"] = "MATCH_ETHERTYPE"; - r["not"] = ((rule.t & 0x80) != 0); - r["etherType"] = (unsigned int)rule.v.etherType; - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - r["type"] = "MATCH_MAC_SOURCE"; - r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - r["type"] = "MATCH_MAC_DEST"; - r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - r["type"] = "MATCH_IPV4_SOURCE"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - r["type"] = "MATCH_IPV4_DEST"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - r["type"] = "MATCH_IPV6_SOURCE"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - r["type"] = "MATCH_IPV6_DEST"; - r["not"] = ((rule.t & 0x80) != 0); - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + default: break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - r["type"] = "MATCH_IP_TOS"; - r["not"] = ((rule.t & 0x80) != 0); - r["ipTos"] = (unsigned int)rule.v.ipTos; - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - r["type"] = "MATCH_IP_PROTOCOL"; - r["not"] = ((rule.t & 0x80) != 0); - r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; - break; - case ZT_NETWORK_RULE_MATCH_ICMP: - r["type"] = "MATCH_ICMP"; - r["not"] = ((rule.t & 0x80) != 0); - r["type"] = (unsigned int)rule.v.icmp.type; - if ((rule.v.icmp.flags & 0x01) != 0) - r["code"] = (unsigned int)rule.v.icmp.code; - else r["code"] = json(); - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; - r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (unsigned int)rule.v.port[0]; - r["end"] = (unsigned int)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - r["type"] = "MATCH_IP_DEST_PORT_RANGE"; - r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (unsigned int)rule.v.port[0]; - r["end"] = (unsigned int)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - r["type"] = "MATCH_CHARACTERISTICS"; - r["not"] = ((rule.t & 0x80) != 0); - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); - r["mask"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - r["type"] = "MATCH_FRAME_SIZE_RANGE"; - r["not"] = ((rule.t & 0x80) != 0); - r["start"] = (unsigned int)rule.v.frameSize[0]; - r["end"] = (unsigned int)rule.v.frameSize[1]; - break; - case ZT_NETWORK_RULE_MATCH_RANDOM: - r["type"] = "MATCH_RANDOM"; - r["not"] = ((rule.t & 0x80) != 0); - r["probability"] = (unsigned long)rule.v.randomProbability; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: - r["type"] = "MATCH_TAGS_DIFFERENCE"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: - r["type"] = "MATCH_TAGS_BITWISE_AND"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: - r["type"] = "MATCH_TAGS_BITWISE_OR"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: - r["type"] = "MATCH_TAGS_BITWISE_XOR"; - r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: - r["type"] = "MATCH_TAGS_EQUAL"; + } + + if (r.size() == 0) { + switch(rt) { + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + r["type"] = "MATCH_VLAN_ID"; + r["vlanId"] = (unsigned int)rule.v.vlanId; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + r["type"] = "MATCH_VLAN_PCP"; + r["vlanPcp"] = (unsigned int)rule.v.vlanPcp; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + r["type"] = "MATCH_VLAN_DEI"; + r["vlanDei"] = (unsigned int)rule.v.vlanDei; + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + r["type"] = "MATCH_MAC_SOURCE"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + r["type"] = "MATCH_MAC_DEST"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + r["type"] = "MATCH_IPV4_SOURCE"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + r["type"] = "MATCH_IPV4_DEST"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + r["type"] = "MATCH_IPV6_SOURCE"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + r["type"] = "MATCH_IPV6_DEST"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + r["type"] = "MATCH_IP_TOS"; + r["ipTos"] = (unsigned int)rule.v.ipTos; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + r["type"] = "MATCH_IP_PROTOCOL"; + r["ipProtocol"] = (unsigned int)rule.v.ipProtocol; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + r["type"] = "MATCH_ETHERTYPE"; + r["etherType"] = (unsigned int)rule.v.etherType; + break; + case ZT_NETWORK_RULE_MATCH_ICMP: + r["type"] = "MATCH_ICMP"; + r["type"] = (unsigned int)rule.v.icmp.type; + if ((rule.v.icmp.flags & 0x01) != 0) + r["code"] = (unsigned int)rule.v.icmp.code; + else r["code"] = json(); + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["start"] = (unsigned int)rule.v.port[0]; + r["end"] = (unsigned int)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["start"] = (unsigned int)rule.v.port[0]; + r["end"] = (unsigned int)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + r["type"] = "MATCH_CHARACTERISTICS"; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); + r["mask"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["start"] = (unsigned int)rule.v.frameSize[0]; + r["end"] = (unsigned int)rule.v.frameSize[1]; + break; + case ZT_NETWORK_RULE_MATCH_RANDOM: + r["type"] = "MATCH_RANDOM"; + r["probability"] = (unsigned long)rule.v.randomProbability; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: + r["type"] = "MATCH_TAGS_DIFFERENCE"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + r["type"] = "MATCH_TAGS_EQUAL"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + default: + break; + } + + if (r.size() > 0) { r["not"] = ((rule.t & 0x80) != 0); - r["id"] = rule.v.tag.id; - r["value"] = rule.v.tag.value; - break; + r["or"] = ((rule.t & 0x40) != 0); + } } + return r; } @@ -301,11 +293,16 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) { if (!r.is_object()) return false; + const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); + if (_jB(r["not"],false)) rule.t = 0x80; else rule.t = 0x00; + if (_jB(r["or"],false)) + rule.t |= 0x40; + if (t == "ACTION_DROP") { rule.t |= ZT_NETWORK_RULE_ACTION_DROP; return true; @@ -352,10 +349,6 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; rule.v.vlanDei = (uint8_t)(_jI(r["vlanDei"],0ULL) & 0xffULL); return true; - } else if (t == "MATCH_ETHERTYPE") { - rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); - return true; } else if (t == "MATCH_MAC_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; const std::string mac(_jS(r["mac"],"0")); @@ -402,6 +395,10 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; rule.v.ipProtocol = (uint8_t)(_jI(r["ipProtocol"],0ULL) & 0xffULL); return true; + } else if (t == "MATCH_ETHERTYPE") { + rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; + rule.v.etherType = (uint16_t)(_jI(r["etherType"],0ULL) & 0xffffULL); + return true; } else if (t == "MATCH_ICMP") { rule.t |= ZT_NETWORK_RULE_MATCH_ICMP; rule.v.icmp.type = (uint8_t)(_jI(r["type"],0ULL) & 0xffULL); @@ -470,6 +467,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) rule.v.tag.value = (uint32_t)(_jI(r["value"],0ULL) & 0xffffffffULL); return true; } + return false; } diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8d7b0cd4..17112e90 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -491,15 +491,15 @@ enum ZT_VirtualNetworkType /** * The type of a virtual network rules table entry * - * These must range from 0 to 127 (0x7f) because the most significant bit - * is reserved as a NOT flag. + * These must be from 0 to 63 since the most significant two bits of each + * rule type are NOT (MSB) and AND/OR. * * Each rule is composed of zero or more MATCHes followed by an ACTION. * An ACTION with no MATCHes is always taken. */ enum ZT_VirtualNetworkRuleType { - // 0 to 31 reserved for actions + // 0 to 15 reserved for actions /** * Drop frame @@ -534,139 +534,40 @@ enum ZT_VirtualNetworkRuleType /** * Maximum ID for an ACTION, anything higher is a MATCH */ - ZT_NETWORK_RULE_ACTION__MAX_ID = 31, - - // 32 to 127 reserved for match criteria - - /** - * Source ZeroTier address -- analogous to an Ethernet port ID on a switch - */ - ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS = 32, - - /** - * Destination ZeroTier address -- analogous to an Ethernet port ID on a switch - */ - ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS = 33, - - /** - * Ethernet VLAN ID - */ - ZT_NETWORK_RULE_MATCH_VLAN_ID = 34, - - /** - * Ethernet VLAN PCP - */ - ZT_NETWORK_RULE_MATCH_VLAN_PCP = 35, - - /** - * Ethernet VLAN DEI - */ - ZT_NETWORK_RULE_MATCH_VLAN_DEI = 36, - - /** - * Ethernet frame type - */ + ZT_NETWORK_RULE_ACTION__MAX_ID = 15, + + // 16 to 63 reserved for match criteria + + ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS = 24, + ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS = 25, + ZT_NETWORK_RULE_MATCH_VLAN_ID = 26, + ZT_NETWORK_RULE_MATCH_VLAN_PCP = 27, + ZT_NETWORK_RULE_MATCH_VLAN_DEI = 28, + ZT_NETWORK_RULE_MATCH_MAC_SOURCE = 29, + ZT_NETWORK_RULE_MATCH_MAC_DEST = 30, + ZT_NETWORK_RULE_MATCH_IPV4_SOURCE = 31, + ZT_NETWORK_RULE_MATCH_IPV4_DEST = 32, + ZT_NETWORK_RULE_MATCH_IPV6_SOURCE = 33, + ZT_NETWORK_RULE_MATCH_IPV6_DEST = 34, + ZT_NETWORK_RULE_MATCH_IP_TOS = 35, + ZT_NETWORK_RULE_MATCH_IP_PROTOCOL = 36, ZT_NETWORK_RULE_MATCH_ETHERTYPE = 37, - - /** - * Source Ethernet MAC address - */ - ZT_NETWORK_RULE_MATCH_MAC_SOURCE = 38, - - /** - * Destination Ethernet MAC address - */ - ZT_NETWORK_RULE_MATCH_MAC_DEST = 39, - - /** - * Source IPv4 address - */ - ZT_NETWORK_RULE_MATCH_IPV4_SOURCE = 40, - - /** - * Destination IPv4 address - */ - ZT_NETWORK_RULE_MATCH_IPV4_DEST = 41, - - /** - * Source IPv6 address - */ - ZT_NETWORK_RULE_MATCH_IPV6_SOURCE = 42, - - /** - * Destination IPv6 address - */ - ZT_NETWORK_RULE_MATCH_IPV6_DEST = 43, - - /** - * IP TOS (type of service) - */ - ZT_NETWORK_RULE_MATCH_IP_TOS = 44, - - /** - * IP protocol - */ - ZT_NETWORK_RULE_MATCH_IP_PROTOCOL = 45, - - /** - * ICMP type and possibly code (does not match if not ICMP) - */ - ZT_NETWORK_RULE_MATCH_ICMP = 46, - - /** - * IP source port range (start-end, inclusive) - */ - ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 47, - - /** - * IP destination port range (start-end, inclusive) - */ - ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 48, - - /** - * Packet characteristics (set of flags) - */ - ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 49, - - /** - * Frame size range (start-end, inclusive) - */ - ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 50, - - /** - * Random match with selectable probability - */ - ZT_NETWORK_RULE_MATCH_RANDOM = 51, - - /** - * Match if local and remote tags differ by no more than value, use 0 to check for equality - */ - ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 52, - - /** - * Match if local and remote tags ANDed together equal value. - */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 53, - - /** - * Match if local and remote tags ANDed together equal value. - */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 54, - - /** - * Match if local and remote tags XORed together equal value. - */ - ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 55, - - /** - * Match if local and remote tags both equal a value - */ - ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 56, + ZT_NETWORK_RULE_MATCH_ICMP = 38, + ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE = 39, + ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE = 40, + ZT_NETWORK_RULE_MATCH_CHARACTERISTICS = 41, + ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE = 42, + ZT_NETWORK_RULE_MATCH_RANDOM = 43, + ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE = 44, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND = 45, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 46, + ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 47, + ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 48, /** * Maximum ID allowed for a MATCH entry in the rules table */ - ZT_NETWORK_RULE_MATCH__MAX_ID = 127 + ZT_NETWORK_RULE_MATCH__MAX_ID = 63 }; /** diff --git a/node/Capability.hpp b/node/Capability.hpp index 99980ce7..2c829ee5 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -166,7 +166,7 @@ public: // field followed by field data. The inclusion of the size will allow non-supported // rules to be ignored but still parsed. b.append((uint8_t)rules[i].t); - switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x7f)) { + switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x3f)) { //case ZT_NETWORK_RULE_ACTION_DROP: //case ZT_NETWORK_RULE_ACTION_ACCEPT: //case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: @@ -198,10 +198,6 @@ public: b.append((uint8_t)1); b.append((uint8_t)rules[i].v.vlanDei); break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - b.append((uint8_t)2); - b.append((uint16_t)rules[i].v.etherType); - break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: b.append((uint8_t)6); @@ -227,6 +223,10 @@ public: b.append((uint8_t)1); b.append((uint8_t)rules[i].v.ipProtocol); break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + b.append((uint8_t)2); + b.append((uint16_t)rules[i].v.etherType); + break; case ZT_NETWORK_RULE_MATCH_ICMP: b.append((uint8_t)3); b.append((uint8_t)rules[i].v.icmp.type); @@ -270,7 +270,7 @@ public: while ((ruleCount < maxRuleCount)&&(p < b.size())) { rules[ruleCount].t = (uint8_t)b[p++]; const unsigned int fieldLen = (unsigned int)b[p++]; - switch((ZT_VirtualNetworkRuleType)(rules[ruleCount].t & 0x7f)) { + switch((ZT_VirtualNetworkRuleType)(rules[ruleCount].t & 0x3f)) { default: break; case ZT_NETWORK_RULE_ACTION_TEE: @@ -293,9 +293,6 @@ public: case ZT_NETWORK_RULE_MATCH_VLAN_DEI: rules[ruleCount].v.vlanDei = (uint8_t)b[p]; break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - rules[ruleCount].v.etherType = b.template at(p); - break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: case ZT_NETWORK_RULE_MATCH_MAC_DEST: memcpy(rules[ruleCount].v.mac,b.field(p,6),6); @@ -316,6 +313,9 @@ public: case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + rules[ruleCount].v.etherType = b.template at(p); + break; case ZT_NETWORK_RULE_MATCH_ICMP: rules[ruleCount].v.icmp.type = (uint8_t)b[p]; rules[ruleCount].v.icmp.code = (uint8_t)b[p+1]; diff --git a/node/Network.cpp b/node/Network.cpp index 00c201ba..177f1a6d 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -58,7 +58,6 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: return "MATCH_VLAN_PCP"; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: return "MATCH_VLAN_DEI"; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: return "MATCH_ETHERTYPE"; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: return "MATCH_MAC_SOURCE"; case ZT_NETWORK_RULE_MATCH_MAC_DEST: return "MATCH_MAC_DEST"; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: return "MATCH_IPV4_SOURCE"; @@ -67,6 +66,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_MATCH_IPV6_DEST: return "MATCH_IPV6_DEST"; case ZT_NETWORK_RULE_MATCH_IP_TOS: return "MATCH_IP_TOS"; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: return "MATCH_IP_PROTOCOL"; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: return "MATCH_ETHERTYPE"; case ZT_NETWORK_RULE_MATCH_ICMP: return "MATCH_ICMP"; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; @@ -182,7 +182,7 @@ static _doZtFilterResult _doZtFilter( uint8_t thisSetMatches = 1; for(unsigned int rn=0;rn %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanDei,0,(unsigned int)thisRuleMatches); break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); - FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); - break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macSource.toInt(),(unsigned int)thisRuleMatches); @@ -380,6 +377,10 @@ static _doZtFilterResult _doZtFilter( FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); + FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); + break; case ZT_NETWORK_RULE_MATCH_ICMP: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { if (frameData[9] == 0x01) { @@ -560,8 +561,9 @@ static _doZtFilterResult _doZtFilter( break; } - // State of equals state AND result of last MATCH (possibly NOTed depending on bit 0x80) - thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); + if ((rules[rn].t & 0x40)) + thisSetMatches |= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); + else thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); } return DOZTFILTER_NO_MATCH; -- cgit v1.2.3 From 93b4ac5cb28408d4bcb63433d9f93a6efe188319 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 13:17:30 -0700 Subject: Remove unused POW code, will revisit later. --- node/IncomingPacket.cpp | 143 ------------------------------------------------ node/IncomingPacket.hpp | 22 -------- node/Packet.cpp | 1 - node/Packet.hpp | 45 --------------- 4 files changed, 211 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e0fa3bf1..5afacd0e 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -106,7 +106,6 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); - case Packet::VERB_REQUEST_PROOF_OF_WORK: return _doREQUEST_PROOF_OF_WORK(RR,peer); case Packet::VERB_USER_MESSAGE: return true; } @@ -1421,70 +1420,6 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S return true; } -bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr &peer) -{ - try { - // If this were allowed from anyone, it would itself be a DOS vector. Right - // now we only allow it from roots and controllers of networks you have joined. - bool allowed = RR->topology->isUpstream(peer->identity()); - if (!allowed) { - std::vector< SharedPtr > allNetworks(RR->node->allNetworks()); - for(std::vector< SharedPtr >::const_iterator n(allNetworks.begin());n!=allNetworks.end();++n) { - if (peer->address() == (*n)->controller()) { - allowed = true; - break; - } - } - } - - if (allowed) { - const uint64_t pid = packetId(); - const unsigned int difficulty = (*this)[ZT_PACKET_IDX_PAYLOAD + 1]; - const unsigned int challengeLength = at(ZT_PACKET_IDX_PAYLOAD + 2); - if (challengeLength > ZT_PROTO_MAX_PACKET_LENGTH) - return true; // sanity check, drop invalid size - const unsigned char *challenge = field(ZT_PACKET_IDX_PAYLOAD + 4,challengeLength); - - switch((*this)[ZT_PACKET_IDX_PAYLOAD]) { - - // Salsa20/12+SHA512 hashcash - case 0x01: { - if (difficulty <= 14) { - unsigned char result[16]; - computeSalsa2012Sha512ProofOfWork(difficulty,challenge,challengeLength,result); - TRACE("PROOF_OF_WORK computed for %s: difficulty==%u, challengeLength==%u, result: %.16llx%.16llx",peer->address().toString().c_str(),difficulty,challengeLength,Utils::ntoh(*(reinterpret_cast(result))),Utils::ntoh(*(reinterpret_cast(result + 8)))); - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_REQUEST_PROOF_OF_WORK); - outp.append(pid); - outp.append((uint16_t)sizeof(result)); - outp.append(result,sizeof(result)); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); - } else { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_REQUEST_PROOF_OF_WORK); - outp.append(pid); - outp.append((unsigned char)Packet::ERROR_INVALID_REQUEST); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); - } - } break; - - default: - TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unrecognized proof of work type",peer->address().toString().c_str(),_path->address().toString().c_str()); - break; - } - - peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false); - } else { - TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_path->address().toString().c_str()); - } - } catch ( ... ) { - TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); - } - return true; -} - void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid) { const uint64_t now = RR->node->now(); @@ -1499,82 +1434,4 @@ void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,cons } } -void IncomingPacket::computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16]) -{ - unsigned char salsabuf[131072]; // 131072 == protocol constant, size of memory buffer for this proof of work function - char candidatebuf[ZT_PROTO_MAX_PACKET_LENGTH + 256]; - unsigned char shabuf[ZT_SHA512_DIGEST_LEN]; - const uint64_t s20iv = 0; // zero IV for Salsa20 - char *const candidate = (char *)(( ((uintptr_t)&(candidatebuf[0])) | 0xf ) + 1); // align to 16-byte boundary to ensure that uint64_t type punning of initial nonce is okay - Salsa20 s20; - unsigned int d; - unsigned char *p; - - Utils::getSecureRandom(candidate,16); - memcpy(candidate + 16,challenge,challengeLength); - - if (difficulty > 512) - difficulty = 512; // sanity check - -try_salsa2012sha512_again: - ++*(reinterpret_cast(candidate)); - - SHA512::hash(shabuf,candidate,16 + challengeLength); - s20.init(shabuf,256,&s20iv); - memset(salsabuf,0,sizeof(salsabuf)); - s20.encrypt12(salsabuf,salsabuf,sizeof(salsabuf)); - SHA512::hash(shabuf,salsabuf,sizeof(salsabuf)); - - d = difficulty; - p = shabuf; - while (d >= 8) { - if (*(p++)) - goto try_salsa2012sha512_again; - d -= 8; - } - if (d > 0) { - if ( ((((unsigned int)*p) << d) & 0xff00) != 0 ) - goto try_salsa2012sha512_again; - } - - memcpy(result,candidate,16); -} - -bool IncomingPacket::testSalsa2012Sha512ProofOfWorkResult(unsigned int difficulty,const void *challenge,unsigned int challengeLength,const unsigned char proposedResult[16]) -{ - unsigned char salsabuf[131072]; // 131072 == protocol constant, size of memory buffer for this proof of work function - char candidate[ZT_PROTO_MAX_PACKET_LENGTH + 256]; - unsigned char shabuf[ZT_SHA512_DIGEST_LEN]; - const uint64_t s20iv = 0; // zero IV for Salsa20 - Salsa20 s20; - unsigned int d; - unsigned char *p; - - if (difficulty > 512) - difficulty = 512; // sanity check - - memcpy(candidate,proposedResult,16); - memcpy(candidate + 16,challenge,challengeLength); - - SHA512::hash(shabuf,candidate,16 + challengeLength); - s20.init(shabuf,256,&s20iv); - memset(salsabuf,0,sizeof(salsabuf)); - s20.encrypt12(salsabuf,salsabuf,sizeof(salsabuf)); - SHA512::hash(shabuf,salsabuf,sizeof(salsabuf)); - - d = difficulty; - p = shabuf; - while (d >= 8) { - if (*(p++)) - return false; - d -= 8; - } - if (d > 0) { - if ( ((((unsigned int)*p) << d) & 0xff00) != 0 ) - return false; - } - - return true; -} - } // namespace ZeroTier diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index c3632216..80244ea4 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -111,27 +111,6 @@ public: */ inline uint64_t receiveTime() const throw() { return _receiveTime; } - /** - * Compute the Salsa20/12+SHA512 proof of work function - * - * @param difficulty Difficulty in bits (max: 64) - * @param challenge Challenge string - * @param challengeLength Length of challenge in bytes (max allowed: ZT_PROTO_MAX_PACKET_LENGTH) - * @param result Buffer to fill with 16-byte result - */ - static void computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16]); - - /** - * Verify the result of Salsa20/12+SHA512 proof of work - * - * @param difficulty Difficulty in bits (max: 64) - * @param challenge Challenge bytes - * @param challengeLength Length of challenge in bytes (max allowed: ZT_PROTO_MAX_PACKET_LENGTH) - * @param proposedResult Result supplied by client - * @return True if result is valid - */ - static bool testSalsa2012Sha512ProofOfWorkResult(unsigned int difficulty,const void *challenge,unsigned int challengeLength,const unsigned char proposedResult[16]); - private: // These are called internally to handle packet contents once it has // been authenticated, decrypted, decompressed, and classified. @@ -152,7 +131,6 @@ private: bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr &peer); void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid); diff --git a/node/Packet.cpp b/node/Packet.cpp index 3b8e1387..20b80962 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -46,7 +46,6 @@ const char *Packet::verbString(Verb v) case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST"; case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT"; - case VERB_REQUEST_PROOF_OF_WORK: return "REQUEST_PROOF_OF_WORK"; case VERB_USER_MESSAGE: return "USER_MESSAGE"; } return "(unknown)"; diff --git a/node/Packet.hpp b/node/Packet.hpp index cc3d323b..a8738884 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -891,8 +891,6 @@ public: */ VERB_MULTICAST_FRAME = 0x0e, - // 0x0f is reserved for an old deprecated message - /** * Push of potential endpoints for direct communication: * <[2] 16-bit number of paths> @@ -1044,49 +1042,6 @@ public: */ VERB_CIRCUIT_TEST_REPORT = 0x12, - /** - * Request proof of work: - * <[1] 8-bit proof of work type> - * <[1] 8-bit proof of work difficulty> - * <[2] 16-bit length of proof of work challenge> - * <[...] proof of work challenge> - * - * This requests that a peer perform a proof of work calucation. It can be - * sent by highly trusted peers (e.g. root servers, network controllers) - * under suspected denial of service conditions in an attempt to filter - * out "non-serious" peers and remain responsive to those proving their - * intent to actually communicate. - * - * If the peer obliges to perform the work, it does so and responds with - * an OK containing the result. Otherwise it may ignore the message or - * response with an ERROR_INVALID_REQUEST or ERROR_UNSUPPORTED_OPERATION. - * - * Proof of work type IDs: - * 0x01 - Salsa20/12+SHA512 hashcash function - * - * Salsa20/12+SHA512 is based on the following composite hash function: - * - * (1) Compute SHA512(candidate) - * (2) Use the first 256 bits of the result of #1 as a key to encrypt - * 131072 zero bytes with Salsa20/12 (with a zero IV). - * (3) Compute SHA512(the result of step #2) - * (4) Accept this candiate if the first [difficulty] bits of the result - * from step #3 are zero. Otherwise generate a new candidate and try - * again. - * - * This is performed repeatedly on candidates generated by appending the - * supplied challenge to an arbitrary nonce until a valid candidate - * is found. This chosen prepended nonce is then returned as the result - * in OK. - * - * OK payload: - * <[2] 16-bit length of result> - * <[...] computed proof of work> - * - * ERROR has no payload. - */ - VERB_REQUEST_PROOF_OF_WORK = 0x13, - /** * A message with arbitrary user-definable content: * <[8] 64-bit arbitrary message type ID> -- cgit v1.2.3 From 2d6a4e5974430ea01cf424aab8b372ebc872b25f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 13:52:45 -0700 Subject: cleanup --- controller/EmbeddedNetworkController.cpp | 22 ---------------------- controller/EmbeddedNetworkController.hpp | 12 ------------ node/Packet.cpp | 2 +- 3 files changed, 1 insertion(+), 35 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 7559cf83..2c43d4b5 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -517,19 +517,6 @@ void EmbeddedNetworkController::threadMain() lastUpdatedNetworkMemberCache = OSUtils::now(); } - { // Every 25ms we push up to 50 network refreshes, which amounts to a max of about 300-500kb/sec - unsigned int count = 0; - Mutex::Lock _l(_refreshQueue_m); - while (_refreshQueue.size() > 0) { - _Refresh &r = _refreshQueue.front(); - //if (_node) - // _node->pushNetworkRefresh(r.dest,r.nwid,r.blacklistAddresses,r.blacklistThresholds,r.numBlacklistEntries); - _refreshQueue.pop_front(); - if (++count >= 50) - break; - } - } - Thread::sleep(25); } } @@ -1230,15 +1217,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _networkMemberCache[nwid][Address(address)] = member; } - { - Mutex::Lock _l(_refreshQueue_m); - _refreshQueue.push_back(_Refresh()); - _Refresh &r = _refreshQueue.back(); - r.dest = Address(address); - r.nwid = nwid; - r.numBlacklistEntries = 0; - } - // Add non-persisted fields member["clock"] = now; diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 1e0fa576..99e1836d 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -204,18 +204,6 @@ private: std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; - // Queue of network member refreshes to be pushed - struct _Refresh - { - Address dest; - uint64_t nwid; - uint64_t blacklistAddresses[64]; - uint64_t blacklistThresholds[64]; - unsigned int numBlacklistEntries; - }; - std::list< _Refresh > _refreshQueue; - Mutex _refreshQueue_m; - Thread _daemon; volatile bool _daemonRun; }; diff --git a/node/Packet.cpp b/node/Packet.cpp index 20b80962..bdbc5fe4 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -40,7 +40,7 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; case VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; - case VERB_NETWORK_CONFIG: return "NETWORK_CONFIG_REFRESH"; + case VERB_NETWORK_CONFIG: return "NETWORK_CONFIG"; case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; -- cgit v1.2.3 From 8850a8610abe1dabd2c035e2e57ccbe1a4aff392 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 13:59:17 -0700 Subject: Fix filter trace. --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 177f1a6d..8a30f5f8 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -499,7 +499,7 @@ static _doZtFilterResult _doZtFilter( } } thisRuleMatches = (uint8_t)((cf | rules[rn].v.characteristics) != 0); - FILTER_TRACE("%u %s %c (%.16llx & %.16llx)==%.16llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics[0],rules[rn].v.characteristics[1],(unsigned int)thisRuleMatches); + FILTER_TRACE("%u %s %c (%.16llx | %.16llx)!=0 -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics,(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); -- cgit v1.2.3 From 4f3775bb866aa4b19024266cb3db1705f3c9f710 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 14:21:00 -0700 Subject: Fix ICMP match. --- node/Network.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 8a30f5f8..4fd3bfc1 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -383,8 +383,8 @@ static _doZtFilterResult _doZtFilter( break; case ZT_NETWORK_RULE_MATCH_ICMP: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - if (frameData[9] == 0x01) { - const unsigned int ihl = (frameData[0] & 0xf) * 32; + if (frameData[9] == 0x01) { // IP protocol == ICMP + const unsigned int ihl = (frameData[0] & 0xf) * 4; if (frameLen >= (ihl + 2)) { if (rules[rn].v.icmp.type == frameData[ihl]) { if ((rules[rn].v.icmp.flags & 0x01) != 0) { -- cgit v1.2.3 From ce6b5bc6f537e8d625f5d9e117b600efae3f82e0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 14:21:24 -0700 Subject: . --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 4fd3bfc1..55509efa 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -36,7 +36,7 @@ #include "Peer.hpp" // Uncomment to make the rules engine dump trace info to stdout -//#define ZT_RULES_ENGINE_DEBUGGING 1 +#define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { -- cgit v1.2.3 From 6469aa9df92ec7820e77867268acbb64d20cc45e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 14:28:39 -0700 Subject: typo --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 55509efa..d888b51c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -417,7 +417,7 @@ static _doZtFilterResult _doZtFilter( } else { thisRuleMatches = 0; } - FILTER_TRACE("%u %s %c (IPv4) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[pos],(int)rules[rn].v.icmp.type,(int)frameData[pos+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); + FILTER_TRACE("%u %s %c (IPv6) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[pos],(int)rules[rn].v.icmp.type,(int)frameData[pos+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; FILTER_TRACE("%u %s %c [frame not ICMPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); -- cgit v1.2.3 From 27d997a2e58b7aa65edf8321275984b72e2bb729 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Oct 2016 15:17:17 -0700 Subject: . --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index d888b51c..c0e4b105 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -36,7 +36,7 @@ #include "Peer.hpp" // Uncomment to make the rules engine dump trace info to stdout -#define ZT_RULES_ENGINE_DEBUGGING 1 +//#define ZT_RULES_ENGINE_DEBUGGING 1 namespace ZeroTier { -- cgit v1.2.3 From 8ffae313fd5042bb4330afd98019e4ee907714cb Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 3 Nov 2016 12:10:50 -0700 Subject: add new files & remove old ones from VS project. Now builds & runs on Windows again --- .gitignore | 2 +- controller/EmbeddedNetworkController.cpp | 3 +++ node/Membership.cpp | 28 ++++++++++++++++--------- node/Multicaster.cpp | 2 +- one.cpp | 6 +----- windows/ZeroTierOne.sln | 12 +++++++++-- windows/ZeroTierOne/ZeroTierOne.vcxproj | 11 ++++++++-- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 27 ++++++++++++++++-------- 8 files changed, 61 insertions(+), 30 deletions(-) (limited to 'node') diff --git a/.gitignore b/.gitignore index b01abc35..d3ef1581 100755 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,7 @@ Thumbs.db /windows/TapDriver6/win7Release /windows/*.db /windows/*.opendb - +enc_temp_folder # *nix/Mac build droppings /build-* diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index d9ec76de..8f5cefd1 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -21,7 +21,10 @@ #include #include #include + +#ifndef _WIN32 #include +#endif #include #include diff --git a/node/Membership.cpp b/node/Membership.cpp index c8fb8e4e..d7c7c0e6 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -232,7 +232,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::_RemoteTag *Membership::_newTag(const uint64_t id) { - _RemoteTag *t; + _RemoteTag *t = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -243,17 +243,21 @@ Membership::_RemoteTag *Membership::_newTag(const uint64_t id) minlr = _remoteTags[i]->lastReceived; } } - t->id = id; - t->lastReceived = 0; - t->revocationThreshold = 0; - t->tag = Tag(); + + if (t) { + t->id = id; + t->lastReceived = 0; + t->revocationThreshold = 0; + t->tag = Tag(); + } + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); return t; } Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) { - _RemoteCapability *c; + _RemoteCapability *c = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -264,10 +268,14 @@ Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) minlr = _remoteCaps[i]->lastReceived; } } - c->id = id; - c->lastReceived = 0; - c->revocationThreshold = 0; - c->cap = Capability(); + + if (c) { + c->id = id; + c->lastReceived = 0; + c->revocationThreshold = 0; + c->cap = Capability(); + } + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); return c; } diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 8743e8f8..17649c7b 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -343,7 +343,7 @@ void Multicaster::clean(uint64_t now) { Mutex::Lock _l(_gatherAuth_m); _GatherAuthKey *k = (_GatherAuthKey *)0; - uint64_t *ts = (uint64_t *)ts; + uint64_t *ts = NULL; Hashtable<_GatherAuthKey,uint64_t>::Iterator i(_gatherAuth); while (i.next(k,ts)) { if ((now - *ts) >= ZT_MULTICAST_CREDENTIAL_EXPIRATON) diff --git a/one.cpp b/one.cpp index 79e8caf8..55cc2e19 100644 --- a/one.cpp +++ b/one.cpp @@ -402,7 +402,7 @@ static int cli(int argc,char **argv) auto &addr = assignedAddresses[j]; if (addr.is_string()) { if (aa.length() > 0) aa.push_back(','); - aa.append(addr); + aa.append(addr.get()); } } } @@ -1194,9 +1194,5 @@ int main(int argc,char **argv) delete zt1Service; zt1Service = (OneService *)0; -#ifdef __UNIX_LIKE__ - OSUtils::rm(pidPath.c_str()); -#endif - return returnValue; } diff --git a/windows/ZeroTierOne.sln b/windows/ZeroTierOne.sln index 68596187..e9a1dfd2 100644 --- a/windows/ZeroTierOne.sln +++ b/windows/ZeroTierOne.sln @@ -1,6 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2012 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZeroTierOne", "ZeroTierOne\ZeroTierOne.vcxproj", "{B00A4957-5977-4AC1-9EF4-571DC27EADA2}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TapDriver6", "TapDriver6\TapDriver6.vcxproj", "{43BA7584-D4DB-4F7C-90FC-E2B18A68A213}" @@ -157,7 +159,8 @@ Global {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|Win32.ActiveCfg = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|Win32.Build.0 = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|Win32.Deploy.0 = Debug|Win32 - {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x64.ActiveCfg = Debug|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x64.ActiveCfg = Debug|x64 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x64.Build.0 = Debug|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x86.ActiveCfg = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x86.Build.0 = Debug|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Debug|x86.Deploy.0 = Debug|Win32 @@ -169,6 +172,7 @@ Global {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|Win32.Build.0 = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|Win32.Deploy.0 = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x64.ActiveCfg = Release|x64 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x64.Build.0 = Release|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x86.ActiveCfg = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x86.Build.0 = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Win7 Release|x86.Deploy.0 = Release|Win32 @@ -392,14 +396,18 @@ Global {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|Mixed Platforms.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|Win32.ActiveCfg = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x64.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x64.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x86.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Debug|x86.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Any CPU.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Any CPU.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Mixed Platforms.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Mixed Platforms.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|Win32.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x64.ActiveCfg = Release|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x64.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x86.ActiveCfg = Release|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win7 Release|x86.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win8 Debug|Any CPU.ActiveCfg = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win8 Debug|Any CPU.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Win8 Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 93da4b1b..b4bf308c 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -19,8 +19,8 @@ + - @@ -39,12 +39,13 @@ + - + @@ -54,10 +55,12 @@ + + @@ -251,6 +254,7 @@ NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + 4996 true @@ -267,6 +271,7 @@ NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) false + 4996 true @@ -290,6 +295,7 @@ AnySuitable Speed true + 4996 true @@ -315,6 +321,7 @@ AnySuitable Speed true + 4996 true diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index eeeb8d2f..337b8cbb 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -55,9 +55,6 @@ {f8a1c208-15b8-4d85-a4cb-11d2b82f2d1e} - - {da28e961-1761-41d8-9a59-65b00dfb1302} - {43f75f84-c70d-4d44-a0ef-28a7a399abd4} @@ -91,6 +88,9 @@ {409ec37e-ff36-4c13-b18d-52d6052e0ca2} + + {3cad34c8-c436-43ae-8323-57803637c832} + @@ -171,9 +171,6 @@ Source Files\ext\http-parser - - Source Files\ext\json-parser - Source Files @@ -192,9 +189,6 @@ Source Files\node - - Source Files\node - Source Files\node @@ -252,6 +246,21 @@ Source Files\osdep + + Source Files\node + + + Source Files\node + + + Source Files\node + + + Source Files\node + + + Source Files\controller + -- cgit v1.2.3 From c61ca1dea2001d8b77bf6b1f44da4246c3088e32 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Nov 2016 16:04:08 -0800 Subject: Keep connections up for netconf stuff as well as frames. --- include/ZeroTierOne.h | 10 ---------- node/Node.cpp | 2 -- node/Peer.cpp | 17 +++++++++++------ node/Peer.hpp | 20 ++------------------ service/ControlPlane.cpp | 4 ---- service/README.md | 2 -- 6 files changed, 13 insertions(+), 42 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 17112e90..d0fef1f1 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1018,16 +1018,6 @@ typedef struct */ uint64_t address; - /** - * Time we last received a unicast frame from this peer - */ - uint64_t lastUnicastFrame; - - /** - * Time we last received a multicast rame from this peer - */ - uint64_t lastMulticastFrame; - /** * Remote major version or -1 if not known */ diff --git a/node/Node.cpp b/node/Node.cpp index db9b8ea0..9314478f 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -405,8 +405,6 @@ ZT_PeerList *Node::peers() const for(std::vector< std::pair< Address,SharedPtr > >::iterator pi(peers.begin());pi!=peers.end();++pi) { ZT_Peer *p = &(pl->peers[pl->peerCount++]); p->address = pi->second->address().toInt(); - p->lastUnicastFrame = pi->second->lastUnicastFrame(); - p->lastMulticastFrame = pi->second->lastMulticastFrame(); if (pi->second->remoteVersionKnown()) { p->versionMajor = pi->second->remoteVersionMajor(); p->versionMinor = pi->second->remoteVersionMinor(); diff --git a/node/Peer.cpp b/node/Peer.cpp index 87882dad..94fb5298 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -42,8 +42,7 @@ static uint32_t _natKeepaliveBuf = 0; Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : _lastReceive(0), - _lastUnicastFrame(0), - _lastMulticastFrame(0), + _lastNontrivialReceive(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), _lastCredentialRequestSent(0), @@ -128,10 +127,16 @@ void Peer::received( #endif _lastReceive = now; - if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME)) - _lastUnicastFrame = now; - else if (verb == Packet::VERB_MULTICAST_FRAME) - _lastMulticastFrame = now; + switch (verb) { + case Packet::VERB_FRAME: + case Packet::VERB_EXT_FRAME: + case Packet::VERB_NETWORK_CONFIG_REQUEST: + case Packet::VERB_NETWORK_CONFIG: + case Packet::VERB_MULTICAST_FRAME: + _lastNontrivialReceive = now; + break; + default: break; + } if (trustEstablished) { _lastTrustEstablishedPacketReceived = now; diff --git a/node/Peer.hpp b/node/Peer.hpp index d0589ccf..be05aa3a 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -226,25 +226,10 @@ public: */ inline bool isAlive(const uint64_t now) const { return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } - /** - * @return Time of most recent unicast frame received - */ - inline uint64_t lastUnicastFrame() const { return _lastUnicastFrame; } - - /** - * @return Time of most recent multicast frame received - */ - inline uint64_t lastMulticastFrame() const { return _lastMulticastFrame; } - - /** - * @return Time of most recent frame of any kind (unicast or multicast) - */ - inline uint64_t lastFrame() const { return std::max(_lastUnicastFrame,_lastMulticastFrame); } - /** * @return True if this peer has sent us real network traffic recently */ - inline uint64_t isActive(uint64_t now) const { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline uint64_t isActive(uint64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return Latency in milliseconds or 0 if unknown @@ -469,8 +454,7 @@ private: uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; uint8_t _remoteClusterOptimal6[16]; uint64_t _lastReceive; // direct or indirect - uint64_t _lastUnicastFrame; - uint64_t _lastMulticastFrame; + uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; uint64_t _lastCredentialRequestSent; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 5c135636..f14bae54 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -222,8 +222,6 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) Utils::snprintf(json,sizeof(json), "%s{\n" "%s\t\"address\": \"%.10llx\",\n" - "%s\t\"lastUnicastFrame\": %llu,\n" - "%s\t\"lastMulticastFrame\": %llu,\n" "%s\t\"versionMajor\": %d,\n" "%s\t\"versionMinor\": %d,\n" "%s\t\"versionRev\": %d,\n" @@ -234,8 +232,6 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) "%s}", prefix, prefix,peer->address, - prefix,peer->lastUnicastFrame, - prefix,peer->lastMulticastFrame, prefix,peer->versionMajor, prefix,peer->versionMinor, prefix,peer->versionRev, diff --git a/service/README.md b/service/README.md index 1c71fbdc..f487f2bc 100644 --- a/service/README.md +++ b/service/README.md @@ -114,8 +114,6 @@ Getting /peer returns an array of peer objects for all current peers. See below - - -- cgit v1.2.3 From 5ebf5077f56442908d4a5ced8d969df9e7de8c4f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 9 Nov 2016 17:11:10 -0800 Subject: Log last meta-data in controller, and ease up just a bit on keepalives. --- controller/EmbeddedNetworkController.cpp | 2 +- node/Constants.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index f91ba533..624c3145 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -622,8 +622,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( member["recentLog"] = recentLog; } - // Update last modified time member["lastModified"] = now; + member["lastRequestMetaData"] = metaData.data(); // If they are not authorized, STOP! if (!authorizedBy) { diff --git a/node/Constants.hpp b/node/Constants.hpp index b7042d5d..6400e289 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -256,12 +256,12 @@ /** * How frequently to send heartbeats over in-use paths */ -#define ZT_PATH_HEARTBEAT_PERIOD 10000 +#define ZT_PATH_HEARTBEAT_PERIOD 14000 /** * Paths are considered inactive if they have not received traffic in this long */ -#define ZT_PATH_ALIVE_TIMEOUT 25000 +#define ZT_PATH_ALIVE_TIMEOUT 45000 /** * Minimum time between attempts to check dead paths to see if they can be re-awakened -- cgit v1.2.3 From 226123ca08ffbb5f4e4f0699b92fb9db08576a66 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 11:54:47 -0800 Subject: Refactor controller to permit sending of pushes as well as just replies to config requests. --- controller/EmbeddedNetworkController.cpp | 77 +++++++++++++++-------- controller/EmbeddedNetworkController.hpp | 14 +++-- node/IncomingPacket.cpp | 84 +------------------------ node/Network.cpp | 105 +++++++++++++------------------ node/Network.hpp | 10 ++- node/NetworkController.hpp | 69 +++++++++++++------- node/Node.cpp | 85 +++++++++++++++++++++++++ node/Node.hpp | 8 ++- 8 files changed, 252 insertions(+), 200 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 624c3145..91b59215 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -470,11 +470,21 @@ EmbeddedNetworkController::~EmbeddedNetworkController() { } -NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) +void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender) { - if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { - return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } + this->_sender = sender; + this->_signingId = signingId; +} + +void EmbeddedNetworkController::request( + uint64_t nwid, + const InetAddress &fromAddr, + uint64_t requestPacketId, + const Identity &identity, + const Dictionary &metaData) +{ + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) + return; const uint64_t now = OSUtils::now(); @@ -483,7 +493,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( Mutex::Lock _l(_lastRequestTime_m); uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) - return NetworkController::NETCONF_QUERY_IGNORE; + return; lrt = now; } @@ -496,8 +506,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( network = _db.get("network",nwids,0); member = _db.get("network",nwids,"member",identity.address().toString(),0); } - if (!network.size()) - return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; + + if (!network.size()) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); + return; + } + + json origMember(member); // for detecting modification later _initMember(member); { @@ -507,10 +522,13 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // a "collision" from being able to auth onto our network in place of an already // known member. try { - if (Identity(haveIdStr.c_str()) != identity) - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + if (Identity(haveIdStr.c_str()) != identity) { + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; + } } catch ( ... ) { - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; } } else { // If we do not yet know this member's identity, learn it. @@ -521,7 +539,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // These are always the same, but make sure they are set member["id"] = identity.address().toString(); member["address"] = member["id"]; - member["nwid"] = network["id"]; + member["nwid"] = nwids; // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; @@ -597,7 +615,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } // Log this request - { + if (requestPacketId) { // only log if this is a request, not for generated pushes json rlEntry = json::object(); rlEntry["ts"] = now; rlEntry["authorized"] = (authorizedBy) ? true : false; @@ -620,22 +638,27 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } member["recentLog"] = recentLog; - } - member["lastModified"] = now; - member["lastRequestMetaData"] = metaData.data(); + // Also only do this on real requests + member["lastRequestMetaData"] = metaData.data(); + } // If they are not authorized, STOP! if (!authorizedBy) { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",identity.address().toString(),member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + if (origMember != member) { + member["lastModified"] = now; + Mutex::Lock _l(_db_m); + _db.put("network",nwids,"member",identity.address().toString(),member); + } + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); + return; } // ------------------------------------------------------------------------- // If we made it this far, they are authorized. // ------------------------------------------------------------------------- + NetworkConfig nc; _NetworkMemberInfo nmi; _getNetworkMemberInfo(now,nwid,nmi); @@ -661,8 +684,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); - for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) + for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + } json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -714,7 +738,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(signingId,identity.address())) + if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) ++nc.capabilityCount; if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) break; @@ -733,7 +757,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) break; nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(signingId)) + if (nc.tags[nc.tagCount].sign(_signingId)) ++nc.tagCount; } } @@ -923,17 +947,20 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); - if (com.sign(signingId)) { + if (com.sign(_signingId)) { nc.com = com; } else { - return NETCONF_QUERY_INTERNAL_SERVER_ERROR; + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); + return; } - { + if (member != origMember) { + member["lastModified"] = now; Mutex::Lock _l(_db_m); _db.put("network",nwids,"member",identity.address().toString(),member); } - return NetworkController::NETCONF_QUERY_OK; + + _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 53d3be0f..79b919b9 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -52,13 +52,14 @@ public: EmbeddedNetworkController(Node *node,const char *dbPath); virtual ~EmbeddedNetworkController(); - virtual NetworkController::ResultCode doNetworkConfigRequest( + virtual void init(const Identity &signingId,Sender *sender); + + virtual void request( + uint64_t nwid, const InetAddress &fromAddr, - const Identity &signingId, + uint64_t requestPacketId, const Identity &identity, - uint64_t nwid, - const Dictionary &metaData, - NetworkConfig &nc); + const Dictionary &metaData); unsigned int handleControlPlaneHttpGET( const std::vector &path, @@ -157,6 +158,9 @@ private: Node *const _node; std::string _path; + NetworkController::Sender *_sender; + Identity _signingId; + struct _CircuitTestEntry { ZT_CircuitTest *test; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 5afacd0e..bde5df71 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -865,92 +865,12 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); const unsigned int hopCount = hops(); const uint64_t requestPacketId = packetId(); - bool trustEstablished = false; if (RR->localNetworkController) { const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); const Dictionary metaData(metaDataBytes,metaDataLength); - - NetworkConfig *netconf = new NetworkConfig(); - try { - switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) { - - case NetworkController::NETCONF_QUERY_OK: { - trustEstablished = true; - Dictionary *dconf = new Dictionary(); - try { - if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) { - uint64_t configUpdateId = RR->node->prng(); - if (!configUpdateId) ++configUpdateId; - const unsigned int totalSize = dconf->sizeBytes(); - unsigned int chunkIndex = 0; - while (chunkIndex < totalSize) { - const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - - const unsigned int sigStart = outp.size(); - outp.append(nwid); - outp.append((uint16_t)chunkLen); - outp.append((const void *)(dconf->data() + chunkIndex),chunkLen); - - outp.append((uint8_t)0); // no flags - outp.append((uint64_t)configUpdateId); - outp.append((uint32_t)totalSize); - outp.append((uint32_t)chunkIndex); - - C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + sigStart,outp.size() - sigStart)); - outp.append((uint8_t)1); - outp.append((uint16_t)ZT_C25519_SIGNATURE_LEN); - outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); - - outp.compress(); - RR->sw->send(outp,true); - chunkIndex += chunkLen; - } - } - delete dconf; - } catch ( ... ) { - delete dconf; - throw; - } - } break; - - case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); - outp.append(nwid); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); - } break; - - case NetworkController::NETCONF_QUERY_ACCESS_DENIED: { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); - outp.append(nwid); - outp.armor(peer->key(),true); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); - } break; - - case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR: - break; - case NetworkController::NETCONF_QUERY_IGNORE: - break; - default: - TRACE("NETWORK_CONFIG_REQUEST failed: invalid return value from NetworkController::doNetworkConfigRequest()"); - break; - } - delete netconf; - } catch ( ... ) { - delete netconf; - throw; - } + RR->localNetworkController->request(nwid,(hopCount > 0) ? InetAddress() : _path->address(),requestPacketId,peer->identity(),metaData); } else { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); @@ -961,7 +881,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons _path->send(RR,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,trustEstablished); + peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); } catch (std::exception &exc) { fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); diff --git a/node/Network.cpp b/node/Network.cpp index c0e4b105..1f8e7ebf 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -599,7 +599,7 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : if (conf.length()) { dconf->load(conf.c_str()); if (nconf->fromDictionary(*dconf)) { - this->_setConfiguration(*nconf,false); + this->setConfiguration(*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } @@ -1015,7 +1015,7 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) } if (nc) { - this->_setConfiguration(*nc,true); + this->setConfiguration(*nc,true); delete nc; return configUpdateId; } else { @@ -1025,6 +1025,46 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) return 0; } +int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) +{ + // _lock is NOT locked when this is called + try { + if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) + return 0; + if (_config == nconf) + return 1; // OK config, but duplicate of what we already have + + ZT_VirtualNetworkConfig ctmp; + bool oldPortInitialized; + { + Mutex::Lock _l(_lock); + _config = nconf; + _lastConfigUpdate = RR->node->now(); + _netconfFailure = NETCONF_FAILURE_NONE; + oldPortInitialized = _portInitialized; + _portInitialized = true; + _externalConfig(&ctmp); + } + _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + + if (saveToDisk) { + Dictionary *d = new Dictionary(); + try { + char n[64]; + Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); + if (nconf.toDictionary(*d,false)) + RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); + } catch ( ... ) {} + delete d; + } + + return 2; // OK and configuration has changed + } catch ( ... ) { + TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); + } + return 0; +} + void Network::requestConfiguration() { const Address ctrl(controller()); @@ -1046,26 +1086,7 @@ void Network::requestConfiguration() if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { - NetworkConfig *nconf = new NetworkConfig(); - try { - switch(RR->localNetworkController->doNetworkConfigRequest(InetAddress(),RR->identity,RR->identity,_id,rmd,*nconf)) { - case NetworkController::NETCONF_QUERY_OK: - this->_setConfiguration(*nconf,true); - break; - case NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND: - this->setNotFound(); - break; - case NetworkController::NETCONF_QUERY_ACCESS_DENIED: - this->setAccessDenied(); - break; - default: - this->setNotFound(); - break; - } - } catch ( ... ) { - this->setNotFound(); - } - delete nconf; + RR->localNetworkController->request(_id,InetAddress(),0xffffffffffffffffULL,RR->identity,rmd); } else { this->setNotFound(); } @@ -1257,46 +1278,6 @@ ZT_VirtualNetworkStatus Network::_status() const } } -int Network::_setConfiguration(const NetworkConfig &nconf,bool saveToDisk) -{ - // _lock is NOT locked when this is called - try { - if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) - return 0; - if (_config == nconf) - return 1; // OK config, but duplicate of what we already have - - ZT_VirtualNetworkConfig ctmp; - bool oldPortInitialized; - { - Mutex::Lock _l(_lock); - _config = nconf; - _lastConfigUpdate = RR->node->now(); - _netconfFailure = NETCONF_FAILURE_NONE; - oldPortInitialized = _portInitialized; - _portInitialized = true; - _externalConfig(&ctmp); - } - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); - - if (saveToDisk) { - Dictionary *d = new Dictionary(); - try { - char n[64]; - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - if (nconf.toDictionary(*d,false)) - RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); - } catch ( ... ) {} - delete d; - } - - return 2; // OK and configuration has changed - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); - } - return 0; -} - void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const { // assumes _lock is locked diff --git a/node/Network.hpp b/node/Network.hpp index 527d3048..1627be58 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -187,6 +187,15 @@ public: */ uint64_t handleConfigChunk(const Packet &chunk,unsigned int ptr); + /** + * Set network configuration + * + * @param nconf Network configuration + * @param saveToDisk Save to disk? Used during loading, should usually be true otherwise. + * @return 0 == bad, 1 == accepted but duplicate/unchanged, 2 == accepted and new + */ + int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); + /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this */ @@ -328,7 +337,6 @@ public: inline void **userPtr() throw() { return &_uPtr; } private: - int _setConfiguration(const NetworkConfig &nconf,bool saveToDisk); ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); diff --git a/node/NetworkController.hpp b/node/NetworkController.hpp index db95dd14..fc5db4af 100644 --- a/node/NetworkController.hpp +++ b/node/NetworkController.hpp @@ -27,7 +27,6 @@ namespace ZeroTier { -class RuntimeEnvironment; class Identity; class Address; struct InetAddress; @@ -38,45 +37,69 @@ struct InetAddress; class NetworkController { public: + enum ErrorCode + { + NC_ERROR_NONE = 0, + NC_ERROR_OBJECT_NOT_FOUND = 1, + NC_ERROR_ACCESS_DENIED = 2, + NC_ERROR_INTERNAL_SERVER_ERROR = 3 + }; + /** - * Return value of doNetworkConfigRequest + * Interface for sender used to send pushes and replies */ - enum ResultCode + class Sender { - NETCONF_QUERY_OK = 0, - NETCONF_QUERY_OBJECT_NOT_FOUND = 1, - NETCONF_QUERY_ACCESS_DENIED = 2, - NETCONF_QUERY_INTERNAL_SERVER_ERROR = 3, - NETCONF_QUERY_IGNORE = 4 + public: + /** + * Send a configuration to a remote peer + * + * @param nwid Network ID + * @param requestPacketId Request packet ID to send OK(NETWORK_CONFIG_REQUEST) or 0 to send NETWORK_CONFIG (push) + * @param destination Destination peer Address + * @param nc Network configuration to send + * @param sendLegacyFormatConfig If true, send an old-format network config + */ + virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) = 0; + + /** + * Send a network configuration request error + * + * @param nwid Network ID + * @param requestPacketId Request packet ID or 0 if none + * @param destination Destination peer Address + * @param errorCode Error code + */ + virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0; }; NetworkController() {} virtual ~NetworkController() {} /** - * Handle a network config request, sending replies if necessary - * - * This call is permitted to block, and may be called concurrently from more - * than one thread. Implementations must use locks if needed. + * Called when this is added to a Node to initialize and supply info * - * On internal server errors, the 'error' field in result can be filled in - * to indicate the error. + * @param signingId Identity for signing of network configurations, certs, etc. + * @param sender Sender implementation for sending replies or config pushes + */ + virtual void init(const Identity &signingId,Sender *sender) = 0; + + /** + * Handle a network configuration request * - * @param fromAddr Originating wire address or null address if packet is not direct (or from self) - * @param signingId Identity that should be used to sign results -- must include private key - * @param identity Originating peer ZeroTier identity * @param nwid 64-bit network ID + * @param fromAddr Originating wire address or null address if packet is not direct (or from self) + * @param requestPacketId Packet ID of request packet or 0 if not initiated by remote request + * @param identity ZeroTier identity of originating peer * @param metaData Meta-data bundled with request (if any) - * @param nc NetworkConfig to fill with results * @return Returns NETCONF_QUERY_OK if result 'nc' is valid, or an error code on error */ - virtual NetworkController::ResultCode doNetworkConfigRequest( + virtual void request( + uint64_t nwid, const InetAddress &fromAddr, - const Identity &signingId, + uint64_t requestPacketId, const Identity &identity, - uint64_t nwid, - const Dictionary &metaData, - NetworkConfig &nc) = 0; + const Dictionary &metaData) = 0; }; } // namespace ZeroTier diff --git a/node/Node.cpp b/node/Node.cpp index 9314478f..69808bcf 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -490,6 +490,7 @@ void Node::clearLocalInterfaceAddresses() void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); + RR->localNetworkController->init(RR->identity,this); } ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) @@ -718,6 +719,90 @@ void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_ RR->topology->setTrustedPaths(reinterpret_cast(networks),ids,count); } +void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) +{ + if (destination == RR->identity.address()) { + SharedPtr n(network(nwid)); + if (!n) return; + n->setConfiguration(nc,true); + } else { + Dictionary *dconf = new Dictionary(); + try { + if (nc.toDictionary(*dconf,sendLegacyFormatConfig)) { + uint64_t configUpdateId = prng(); + if (!configUpdateId) ++configUpdateId; + + const unsigned int totalSize = dconf->sizeBytes(); + unsigned int chunkIndex = 0; + while (chunkIndex < totalSize) { + const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); + Packet outp(destination,RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + + const unsigned int sigStart = outp.size(); + outp.append(nwid); + outp.append((uint16_t)chunkLen); + outp.append((const void *)(dconf->data() + chunkIndex),chunkLen); + + outp.append((uint8_t)0); // no flags + outp.append((uint64_t)configUpdateId); + outp.append((uint32_t)totalSize); + outp.append((uint32_t)chunkIndex); + + C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + sigStart,outp.size() - sigStart)); + outp.append((uint8_t)1); + outp.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); + + outp.compress(); + RR->sw->send(outp,true); + chunkIndex += chunkLen; + } + } + delete dconf; + } catch ( ... ) { + delete dconf; + throw; + } + } +} + +void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) +{ + if (destination == RR->identity.address()) { + SharedPtr n(network(nwid)); + if (!n) return; + switch(errorCode) { + case NetworkController::NC_ERROR_OBJECT_NOT_FOUND: + case NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR: + n->setNotFound(); + break; + case NetworkController::NC_ERROR_ACCESS_DENIED: + n->setAccessDenied(); + break; + + default: break; + } + } else if (requestPacketId) { + Packet outp(destination,RR->identity.address(),Packet::VERB_ERROR); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + switch(errorCode) { + //case NetworkController::NC_ERROR_OBJECT_NOT_FOUND: + //case NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR: + default: + outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND); + break; + case NetworkController::NC_ERROR_ACCESS_DENIED: + outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_); + break; + } + outp.append(nwid); + RR->sw->send(outp,true); + } +} + } // namespace ZeroTier /****************************************************************************/ diff --git a/node/Node.hpp b/node/Node.hpp index 11462531..e616da3d 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -36,6 +36,7 @@ #include "Network.hpp" #include "Path.hpp" #include "Salsa20.hpp" +#include "NetworkController.hpp" #undef TRACE #ifdef ZT_TRACE @@ -55,7 +56,7 @@ namespace ZeroTier { * * The pointer returned by ZT_Node_new() is an instance of this class. */ -class Node +class Node : public NetworkController::Sender { public: Node( @@ -69,7 +70,7 @@ public: ZT_PathCheckFunction pathCheckFunction, ZT_EventCallback eventCallback); - ~Node(); + virtual ~Node(); // Public API Functions ---------------------------------------------------- @@ -282,6 +283,9 @@ public: return false; } + virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig); + virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); + private: inline SharedPtr _network(uint64_t nwid) const { -- cgit v1.2.3 From 12d32b9311d86a04353c8182e5d7cf4ec514d3df Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 10 Nov 2016 11:57:45 -0800 Subject: Small fix to send pushes if not a reply. --- node/Node.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 69808bcf..c05a1850 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -736,9 +736,11 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de unsigned int chunkIndex = 0; while (chunkIndex < totalSize) { const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); - Packet outp(destination,RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); + Packet outp(destination,RR->identity.address(),(requestPacketId) ? Packet::VERB_OK : Packet::VERB_NETWORK_CONFIG); + if (requestPacketId) { + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + } const unsigned int sigStart = outp.size(); outp.append(nwid); @@ -800,7 +802,7 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des } outp.append(nwid); RR->sw->send(outp,true); - } + } // else we can't send an ERROR() in response to nothing, so discard } } // namespace ZeroTier -- cgit v1.2.3 From bf8d71e82c27eae1e47bde411054f5258df29146 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 17 Nov 2016 16:20:41 -0800 Subject: Add notion of upstream that is separate from root in Topology, etc. --- attic/CertificateOfTrust.cpp | 67 +++++++++++++++++++ attic/CertificateOfTrust.hpp | 155 +++++++++++++++++++++++++++++++++++++++++++ node/IncomingPacket.cpp | 10 +-- node/Packet.hpp | 19 ++++-- node/Topology.cpp | 87 ++++++++++++++++-------- node/Topology.hpp | 37 ++++++----- objects.mk | 1 + 7 files changed, 321 insertions(+), 55 deletions(-) create mode 100644 attic/CertificateOfTrust.cpp create mode 100644 attic/CertificateOfTrust.hpp (limited to 'node') diff --git a/attic/CertificateOfTrust.cpp b/attic/CertificateOfTrust.cpp new file mode 100644 index 00000000..e85a91df --- /dev/null +++ b/attic/CertificateOfTrust.cpp @@ -0,0 +1,67 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "CertificateOfTrust.hpp" + +#include "RuntimeEnvironment.hpp" +#include "Topology.hpp" +#include "Switch.hpp" + +namespace ZeroTier { + +bool CertificateOfTrust::create(uint64_t ts,uint64_t rls,const Identity &iss,const Identity &tgt,Level l) +{ + if ((!iss)||(!iss.hasPrivate())) + return false; + + _timestamp = ts; + _roles = rls; + _issuer = iss.address(); + _target = tgt; + _level = l; + + Buffer tmp; + tmp.append(_timestamp); + tmp.append(_roles); + _issuer.appendTo(tmp); + _target.serialize(tmp,false); + tmp.append((uint16_t)_level); + _signature = iss.sign(tmp.data(),tmp.size()); + + return true; +} + +int CertificateOfTrust::verify(const RuntimeEnvironment *RR) const +{ + const Identity id(RR->topology->getIdentity(_issuer)); + if (!id) { + RR->sw->requestWhois(_issuer); + return 1; + } + + Buffer tmp; + tmp.append(_timestamp); + tmp.append(_roles); + _issuer.appendTo(tmp); + _target.serialize(tmp,false); + tmp.append((uint16_t)_level); + + return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1); +} + +} // namespace ZeroTier diff --git a/attic/CertificateOfTrust.hpp b/attic/CertificateOfTrust.hpp new file mode 100644 index 00000000..6e3c8743 --- /dev/null +++ b/attic/CertificateOfTrust.hpp @@ -0,0 +1,155 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CERTIFICATEOFTRUST_HPP +#define ZT_CERTIFICATEOFTRUST_HPP + +#include "Constants.hpp" +#include "Identity.hpp" +#include "C25519.hpp" +#include "Buffer.hpp" + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * Certificate of peer to peer trust + */ +class CertificateOfTrust +{ +public: + /** + * Trust levels, with 0 indicating anti-trust + */ + enum Level + { + /** + * Negative trust is reserved for informing peers that another peer is misbehaving, etc. Not currently used. + */ + LEVEL_NEGATIVE = 0, + + /** + * Default trust -- for most peers + */ + LEVEL_DEFAULT = 1, + + /** + * Above normal trust, e.g. common network membership + */ + LEVEL_MEDIUM = 25, + + /** + * High trust -- e.g. an upstream or a controller + */ + LEVEL_HIGH = 50, + + /** + * Right now ultimate is only for roots + */ + LEVEL_ULTIMATE = 100 + }; + + /** + * Role bit masks + */ + enum Role + { + /** + * Target is permitted to represent issuer on the network as a federated root / relay + */ + ROLE_UPSTREAM = 0x00000001 + }; + + CertificateOfTrust() : + _timestamp(0), + _roles(0), + _issuer(), + _target(), + _level(LEVEL_DEFAULT), + _signature() {} + + /** + * Create and sign this certificate of trust + * + * @param ts Cert timestamp + * @param rls Roles bitmap + * @param iss Issuer identity (must have secret key!) + * @param tgt Target identity + * @param l Trust level + * @return True on successful signature + */ + bool create(uint64_t ts,uint64_t rls,const Identity &iss,const Identity &tgt,Level l); + + /** + * Verify this COT and its signature + * + * @param RR Runtime environment for looking up peers + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential + */ + int verify(const RuntimeEnvironment *RR) const; + + inline bool roleUpstream() const { return ((_roles & (uint64_t)ROLE_UPSTREAM) != 0); } + + inline uint64_t timestamp() const { return _timestamp; } + inline uint64_t roles() const { return _roles; } + inline const Address &issuer() const { return _issuer; } + inline const Identity &target() const { return _target; } + inline Level level() const { return _level; } + + inline operator bool() const { return (_issuer); } + + template + inline void serialize(Buffer &b) const + { + b.append(_timestamp); + b.append(_roles); + _issuer.appendTo(b); + _target.serialize(b); + b.append((uint16_t)_level); + b.append((uint8_t)1); // 1 == ed25519 signature + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); + b.append((uint16_t)0); // length of additional fields + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + unsigned int p = startAt; + _timestamp = b.template at(p); p += 8; + _roles = b.template at(p); p += 8; + _issuer.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; + p += _target.deserialize(b,p); + _level = b.template at(p); p += 2; + p += b.template at(p); p += 2; + return (p - startAt); + } + +private: + uint64_t _timestamp; + uint64_t _roles; + Address _issuer; + Identity _target; + Level _level; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index bde5df71..c6346346 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -160,7 +160,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_IDENTITY_COLLISION: // FIXME: for federation this will need a payload with a signature or something. - if (RR->topology->isRoot(peer->identity())) + if (RR->topology->isUpstream(peer->identity())) RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; @@ -508,11 +508,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr id.serialize(outp,false); ++count; } else { - // If I am not the root and don't know this identity, ask upstream. Downstream - // peer may re-request in the future and if so we will be able to provide it. - if (!RR->topology->amRoot()) - RR->sw->requestWhois(addr); - + RR->sw->requestWhois(addr); #ifdef ZT_ENABLE_CLUSTER // Distribute WHOIS queries across a cluster if we do not know the ID. // This may result in duplicate OKs to the querying peer, which is fine. @@ -666,7 +662,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

address(),RR->identity.address(),Packet::VERB_OK); outp.append((uint8_t)Packet::VERB_EXT_FRAME); outp.append((uint64_t)packetId()); diff --git a/node/Packet.hpp b/node/Packet.hpp index a8738884..7a742aad 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -617,10 +617,8 @@ public: * <[1] protocol address length (4 for IPv4, 16 for IPv6)> * <[...] protocol address (network byte order)> * - * This is sent by a relaying node to initiate NAT traversal between two - * peers that are communicating by way of indirect relay. The relay will - * send this to both peers at the same time on a periodic basis, telling - * each where it might find the other on the network. + * An upstream node can send this to inform both sides of a relay of + * information they might use to establish a direct connection. * * Upon receipt a peer sends HELLO to establish a direct link. * @@ -1051,7 +1049,18 @@ public: * OK or ERROR and has no special semantics outside of whatever the user * (via the ZeroTier core API) chooses to give it. */ - VERB_USER_MESSAGE = 0x14 + VERB_USER_MESSAGE = 0x14, + + /** + * Information related to federation and mesh-like behavior: + * <[2] 16-bit length of Dictionary> + * <[...] topology definition info Dictionary> + * + * This message can carry information that can be used to define topology + * and implement "mesh-like" behavior. It can optionally generate OK or + * ERROR, and these carry the same payload. + */ + VERB_TOPOLOGY_HINT = 0x15 }; /** diff --git a/node/Topology.cpp b/node/Topology.cpp index 12a7cc0b..48ced7c5 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -111,9 +111,8 @@ SharedPtr Topology::getPeer(const Address &zta) { Mutex::Lock _l(_lock); const SharedPtr *const ap = _peers.get(zta); - if (ap) { + if (ap) return *ap; - } } try { @@ -158,7 +157,7 @@ void Topology::saveIdentity(const Identity &id) } } -SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCount,bool strictAvoid) +SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid) { const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); @@ -189,22 +188,25 @@ SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCou const SharedPtr *bestOverall = (const SharedPtr *)0; const SharedPtr *bestNotAvoid = (const SharedPtr *)0; - for(std::vector< SharedPtr >::const_iterator r(_rootPeers.begin());r!=_rootPeers.end();++r) { - bool avoiding = false; - for(unsigned int i=0;iaddress()) { - avoiding = true; - break; + for(std::vector

::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { + const SharedPtr *const p = _peers.get(*a); + if (p) { + bool avoiding = false; + for(unsigned int i=0;iaddress()) { + avoiding = true; + break; + } + } + const unsigned int q = (*p)->relayQuality(now); + if (q <= bestQualityOverall) { + bestQualityOverall = q; + bestOverall = &(*p); + } + if ((!avoiding)&&(q <= bestQualityNotAvoid)) { + bestQualityNotAvoid = q; + bestNotAvoid = &(*p); } - } - const unsigned int q = (*r)->relayQuality(now); - if (q <= bestQualityOverall) { - bestQualityOverall = q; - bestOverall = &(*r); - } - if ((!avoiding)&&(q <= bestQualityNotAvoid)) { - bestQualityNotAvoid = q; - bestNotAvoid = &(*r); } } @@ -219,9 +221,34 @@ SharedPtr Topology::getBestRoot(const Address *avoid,unsigned int avoidCou return SharedPtr(); } +bool Topology::isRoot(const Identity &id) const +{ + Mutex::Lock _l(_lock); + return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end()); +} + bool Topology::isUpstream(const Identity &id) const { - return isRoot(id); + Mutex::Lock _l(_lock); + return (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),id.address()) != _upstreamAddresses.end()); +} + +void Topology::setUpstream(const Address &a,bool upstream) +{ + Mutex::Lock _l(_lock); + if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) { + if (upstream) { + if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) + _upstreamAddresses.push_back(a); + } else { + std::vector
ua; + for(std::vector
::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) { + if (a != *i) + ua.push_back(*i); + } + _upstreamAddresses.swap(ua); + } + } } bool Topology::worldUpdateIfValid(const World &newWorld) @@ -249,7 +276,7 @@ void Topology::clean(uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if ( (!(*p)->isAlive(now)) && (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) ) + if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) _peers.erase(*a); } } @@ -280,25 +307,33 @@ Identity Topology::_getIdentity(const Address &zta) void Topology::_setWorld(const World &newWorld) { // assumed _lock is locked (or in constructor) + + std::vector
ua; + for(std::vector
::iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { + if (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) + ua.push_back(*a); + } + _world = newWorld; - _amRoot = false; _rootAddresses.clear(); - _rootPeers.clear(); + _amRoot = false; + for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { _rootAddresses.push_back(r->identity.address()); + if (std::find(ua.begin(),ua.end(),r->identity.address()) == ua.end()) + ua.push_back(r->identity.address()); if (r->identity.address() == RR->identity.address()) { _amRoot = true; } else { SharedPtr *rp = _peers.get(r->identity.address()); - if (rp) { - _rootPeers.push_back(*rp); - } else { + if (!rp) { SharedPtr newrp(new Peer(RR,RR->identity,r->identity)); _peers.set(r->identity.address(),newrp); - _rootPeers.push_back(newrp); } } } + + _upstreamAddresses.swap(ua); } } // namespace ZeroTier diff --git a/node/Topology.hpp b/node/Topology.hpp index e63766cb..573d5ca2 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -125,35 +125,27 @@ public: void saveIdentity(const Identity &id); /** - * Get the current favorite root server + * Get the current best upstream peer * * @return Root server with lowest latency or NULL if none */ - inline SharedPtr getBestRoot() { return getBestRoot((const Address *)0,0,false); } + inline SharedPtr getUpstreamPeer() { return getUpstreamPeer((const Address *)0,0,false); } /** - * Get the best root server, avoiding root servers listed in an array - * - * This will get the best root server (lowest latency, etc.) but will - * try to avoid the listed root servers, only using them if no others - * are available. + * Get the current best upstream peer, avoiding those in the supplied avoid list * * @param avoid Nodes to avoid * @param avoidCount Number of nodes to avoid * @param strictAvoid If false, consider avoided root servers anyway if no non-avoid root servers are available * @return Root server or NULL if none available */ - SharedPtr getBestRoot(const Address *avoid,unsigned int avoidCount,bool strictAvoid); + SharedPtr getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid); /** * @param id Identity to check * @return True if this is a designated root server in this world */ - inline bool isRoot(const Identity &id) const - { - Mutex::Lock _l(_lock); - return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end()); - } + bool isRoot(const Identity &id) const; /** * @param id Identity to check @@ -161,6 +153,16 @@ public: */ bool isUpstream(const Identity &id) const; + /** + * Set whether or not an address is upstream + * + * If the address is a root this does nothing, since roots are fixed. + * + * @param a Target address + * @param upstream New upstream status + */ + void setUpstream(const Address &a,bool upstream); + /** * @return Vector of root server addresses */ @@ -175,7 +177,8 @@ public: */ inline std::vector
upstreamAddresses() const { - return rootAddresses(); + Mutex::Lock _l(_lock); + return _upstreamAddresses; } /** @@ -342,9 +345,9 @@ private: Hashtable< Address,SharedPtr > _peers; Hashtable< Path::HashKey,SharedPtr > _paths; - std::vector< Address > _rootAddresses; - std::vector< SharedPtr > _rootPeers; - bool _amRoot; + std::vector< Address > _upstreamAddresses; // includes roots + std::vector< Address > _rootAddresses; // only roots + bool _amRoot; // am I a root? Mutex _lock; }; diff --git a/objects.mk b/objects.mk index 078a92a7..16858ef3 100644 --- a/objects.mk +++ b/objects.mk @@ -4,6 +4,7 @@ OBJS=\ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ + node/CertificateOfTrust.o \ node/Cluster.o \ node/Identity.o \ node/IncomingPacket.o \ -- cgit v1.2.3 From 1615ef1114de4627e6952d96c62c1d561f8bd03c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 17 Nov 2016 16:31:58 -0800 Subject: Rename getBestRoot() etc. --- node/Multicaster.cpp | 2 +- node/Node.cpp | 8 ++------ node/Packet.hpp | 13 +------------ node/Switch.cpp | 10 +++++----- node/Topology.hpp | 9 --------- objects.mk | 1 - 6 files changed, 9 insertions(+), 34 deletions(-) (limited to 'node') diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 17649c7b..f8d58501 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -229,7 +229,7 @@ void Multicaster::send( Address explicitGatherPeers[16]; unsigned int numExplicitGatherPeers = 0; - SharedPtr bestRoot(RR->topology->getBestRoot()); + SharedPtr bestRoot(RR->topology->getUpstreamPeer()); if (bestRoot) explicitGatherPeers[numExplicitGatherPeers++] = bestRoot->address(); explicitGatherPeers[numExplicitGatherPeers++] = Network::controllerFor(nwid); diff --git a/node/Node.cpp b/node/Node.cpp index c05a1850..add3117e 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -211,8 +211,7 @@ public: } if (upstream) { - // "Upstream" devices are roots and relays and get special treatment -- they stay alive - // forever and we try to keep (if available) both IPv4 and IPv6 channels open to them. + // We keep connections to upstream peers alive forever. bool needToContactIndirect = true; if (p->doPingAndKeepalive(_now,AF_INET)) { needToContactIndirect = false; @@ -231,11 +230,8 @@ public: } } + // If we don't have a direct path or a static endpoint, send something indirectly to find one. if (needToContactIndirect) { - // If this is an upstream and we have no stable endpoint for either IPv4 or IPv6, - // send a NOP indirectly if possible to see if we can get to this peer in any - // way whatsoever. This will e.g. find network preferred relays that lack - // stable endpoints by using root servers. Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP); RR->sw->send(outp,true); } diff --git a/node/Packet.hpp b/node/Packet.hpp index 7a742aad..8ff817aa 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1049,18 +1049,7 @@ public: * OK or ERROR and has no special semantics outside of whatever the user * (via the ZeroTier core API) chooses to give it. */ - VERB_USER_MESSAGE = 0x14, - - /** - * Information related to federation and mesh-like behavior: - * <[2] 16-bit length of Dictionary> - * <[...] topology definition info Dictionary> - * - * This message can carry information that can be used to define topology - * and implement "mesh-like" behavior. It can optionally generate OK or - * ERROR, and these carry the same payload. - */ - VERB_TOPOLOGY_HINT = 0x15 + VERB_USER_MESSAGE = 0x14 }; /** diff --git a/node/Switch.cpp b/node/Switch.cpp index 82b13483..7400fd62 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -131,8 +131,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } #endif - // Don't know peer or no direct path -- so relay via root server - relayTo = RR->topology->getBestRoot(); + // Don't know peer or no direct path -- so relay via someone upstream + relayTo = RR->topology->getUpstreamPeer(); if (relayTo) relayTo->sendDirect(fragment.data(),fragment.size(),now,true); } @@ -254,7 +254,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from return; } #endif - relayTo = RR->topology->getBestRoot(&source,1,true); + relayTo = RR->topology->getUpstreamPeer(&source,1,true); if (relayTo) relayTo->sendDirect(packet.data(),packet.size(),now,true); } @@ -763,7 +763,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) { - SharedPtr upstream(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); + SharedPtr upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); if (upstream) { Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); @@ -793,7 +793,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) viaPath.zero(); } if (!viaPath) { - SharedPtr relay(RR->topology->getBestRoot()); + SharedPtr relay(RR->topology->getUpstreamPeer()); if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { if (!(viaPath = peer->getBestPath(now,true))) return false; diff --git a/node/Topology.hpp b/node/Topology.hpp index 573d5ca2..8e1d28cb 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -163,15 +163,6 @@ public: */ void setUpstream(const Address &a,bool upstream); - /** - * @return Vector of root server addresses - */ - inline std::vector
rootAddresses() const - { - Mutex::Lock _l(_lock); - return _rootAddresses; - } - /** * @return Vector of active upstream addresses (including roots) */ diff --git a/objects.mk b/objects.mk index 16858ef3..078a92a7 100644 --- a/objects.mk +++ b/objects.mk @@ -4,7 +4,6 @@ OBJS=\ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ - node/CertificateOfTrust.o \ node/Cluster.o \ node/Identity.o \ node/IncomingPacket.o \ -- cgit v1.2.3 From 39333c9e8ef3a11fdf2ccad3a5bc7c17ce697bd6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 17 Nov 2016 16:59:04 -0800 Subject: Modify unite() to deal with a second layer of upstreams. --- node/Peer.hpp | 20 ------- node/Switch.cpp | 163 ++++++++++++++++++++++++++++++++------------------------ node/Switch.hpp | 12 +---- 3 files changed, 94 insertions(+), 101 deletions(-) (limited to 'node') diff --git a/node/Peer.hpp b/node/Peer.hpp index be05aa3a..a7240cb4 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -403,26 +403,6 @@ public: return false; } - /** - * Find a common set of addresses by which two peers can link, if any - * - * @param a Peer A - * @param b Peer B - * @param now Current time - * @return Pair: B's address (to send to A), A's address (to send to B) - */ - static inline std::pair findCommonGround(const Peer &a,const Peer &b,uint64_t now) - { - std::pair v4,v6; - b.getBestActiveAddresses(now,v4.first,v6.first); - a.getBestActiveAddresses(now,v4.second,v6.second); - if ((v6.first)&&(v6.second)) // prefer IPv6 if both have it since NAT-t is (almost) unnecessary - return v6; - else if ((v4.first)&&(v4.second)) - return v4; - else return std::pair(); - } - private: inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const { diff --git a/node/Switch.cpp b/node/Switch.cpp index 7400fd62..a5dd57e4 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -237,7 +237,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) { luts = now; - unite(source,destination); + _unite(source,destination); } } else { #ifdef ZT_ENABLE_CLUSTER @@ -590,75 +590,6 @@ void Switch::send(const Packet &packet,bool encrypt) } } -bool Switch::unite(const Address &p1,const Address &p2) -{ - if ((p1 == RR->identity.address())||(p2 == RR->identity.address())) - return false; - SharedPtr p1p = RR->topology->getPeer(p1); - if (!p1p) - return false; - SharedPtr p2p = RR->topology->getPeer(p2); - if (!p2p) - return false; - - const uint64_t now = RR->node->now(); - - std::pair cg(Peer::findCommonGround(*p1p,*p2p,now)); - if ((!(cg.first))||(cg.first.ipScope() != cg.second.ipScope())) - return false; - - TRACE("unite: %s(%s) <> %s(%s)",p1.toString().c_str(),cg.second.toString().c_str(),p2.toString().c_str(),cg.first.toString().c_str()); - - /* Tell P1 where to find P2 and vice versa, sending the packets to P1 and - * P2 in randomized order in terms of which gets sent first. This is done - * since in a few cases NAT-t can be sensitive to slight timing differences - * in terms of when the two peers initiate. Normally this is accounted for - * by the nearly-simultaneous RENDEZVOUS kickoff from the relay, but - * given that relay are hosted on cloud providers this can in some - * cases have a few ms of latency between packet departures. By randomizing - * the order we make each attempted NAT-t favor one or the other going - * first, meaning if it doesn't succeed the first time it might the second - * and so forth. */ - unsigned int alt = (unsigned int)RR->node->prng() & 1; - unsigned int completed = alt + 2; - while (alt != completed) { - if ((alt & 1) == 0) { - // Tell p1 where to find p2. - Packet outp(p1,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((unsigned char)0); - p2.appendTo(outp); - outp.append((uint16_t)cg.first.port()); - if (cg.first.isV6()) { - outp.append((unsigned char)16); - outp.append(cg.first.rawIpData(),16); - } else { - outp.append((unsigned char)4); - outp.append(cg.first.rawIpData(),4); - } - outp.armor(p1p->key(),true); - p1p->sendDirect(outp.data(),outp.size(),now,true); - } else { - // Tell p2 where to find p1. - Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((unsigned char)0); - p1.appendTo(outp); - outp.append((uint16_t)cg.second.port()); - if (cg.second.isV6()) { - outp.append((unsigned char)16); - outp.append(cg.second.rawIpData(),16); - } else { - outp.append((unsigned char)4); - outp.append(cg.second.rawIpData(),4); - } - outp.armor(p2p->key(),true); - p2p->sendDirect(outp.data(),outp.size(),now,true); - } - ++alt; // counts up and also flips LSB - } - - return true; -} - void Switch::requestWhois(const Address &addr) { bool inserted = false; @@ -839,4 +770,96 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) return false; } +bool Switch::_unite(const Address &p1,const Address &p2) +{ + if ((p1 == RR->identity.address())||(p2 == RR->identity.address())) + return false; + + const uint64_t now = RR->node->now(); + InetAddress *p1a = (InetAddress *)0; + InetAddress *p2a = (InetAddress *)0; + InetAddress p1v4,p1v6,p2v4,p2v6,uv4,uv6; + { + const SharedPtr p1p(RR->topology->getPeer(p1)); + const SharedPtr p2p(RR->topology->getPeer(p2)); + if ((!p1p)&&(!p2p)) return false; + if (p1p) p1p->getBestActiveAddresses(now,p1v4,p1v6); + if (p2p) p2p->getBestActiveAddresses(now,p2v4,p2v6); + } + if ((p1v6)&&(p2v6)) { + p1a = &p1v6; + p2a = &p2v6; + } else if ((p1v4)&&(p2v4)) { + p1a = &p1v4; + p2a = &p2v4; + } else { + SharedPtr upstream(RR->topology->getUpstreamPeer()); + if (!upstream) + return false; + upstream->getBestActiveAddresses(now,uv4,uv6); + if ((p1v6)&&(uv6)) { + p1a = &p1v6; + p2a = &uv6; + } else if ((p1v4)&&(uv4)) { + p1a = &p1v4; + p2a = &uv4; + } else if ((p2v6)&&(uv6)) { + p1a = &p2v6; + p2a = &uv6; + } else if ((p2v4)&&(uv4)) { + p1a = &p2v4; + p2a = &uv4; + } else return false; + } + + TRACE("unite: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); + + /* Tell P1 where to find P2 and vice versa, sending the packets to P1 and + * P2 in randomized order in terms of which gets sent first. This is done + * since in a few cases NAT-t can be sensitive to slight timing differences + * in terms of when the two peers initiate. Normally this is accounted for + * by the nearly-simultaneous RENDEZVOUS kickoff from the relay, but + * given that relay are hosted on cloud providers this can in some + * cases have a few ms of latency between packet departures. By randomizing + * the order we make each attempted NAT-t favor one or the other going + * first, meaning if it doesn't succeed the first time it might the second + * and so forth. */ + unsigned int alt = (unsigned int)RR->node->prng() & 1; + const unsigned int completed = alt + 2; + while (alt != completed) { + if ((alt & 1) == 0) { + // Tell p1 where to find p2. + Packet outp(p1,RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((unsigned char)0); + p2.appendTo(outp); + outp.append((uint16_t)p2a->port()); + if (p2a->isV6()) { + outp.append((unsigned char)16); + outp.append(p2a->rawIpData(),16); + } else { + outp.append((unsigned char)4); + outp.append(p2a->rawIpData(),4); + } + send(outp,true); + } else { + // Tell p2 where to find p1. + Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((unsigned char)0); + p1.appendTo(outp); + outp.append((uint16_t)p1a->port()); + if (p1a->isV6()) { + outp.append((unsigned char)16); + outp.append(p1a->rawIpData(),16); + } else { + outp.append((unsigned char)4); + outp.append(p1a->rawIpData(),4); + } + send(outp,true); + } + ++alt; // counts up and also flips LSB + } + + return true; +} + } // namespace ZeroTier diff --git a/node/Switch.hpp b/node/Switch.hpp index 7c903ef9..f44eef48 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -97,17 +97,6 @@ public: */ void send(const Packet &packet,bool encrypt); - /** - * Send RENDEZVOUS to two peers to permit them to directly connect - * - * This only works if both peers are known, with known working direct - * links to this peer. The best link for each peer is sent to the other. - * - * @param p1 One of two peers (order doesn't matter) - * @param p2 Second of pair - */ - bool unite(const Address &p1,const Address &p2); - /** * Request WHOIS on a given address * @@ -138,6 +127,7 @@ public: private: Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); bool _trySend(const Packet &packet,bool encrypt); + bool _unite(const Address &p1,const Address &p2); const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; -- cgit v1.2.3 From 1fcbb1fbedad2d0aff567a0dda84a0985ba063cb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 10:39:26 -0800 Subject: Proactively auto-load designated upstreams. --- node/Topology.cpp | 65 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 23 deletions(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index 48ced7c5..81382e05 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -23,6 +23,7 @@ #include "Network.hpp" #include "NetworkConfig.hpp" #include "Buffer.hpp" +#include "Switch.hpp" namespace ZeroTier { @@ -180,8 +181,8 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi } } else { - /* If I am not a root server, the best root server is the active one with - * the lowest quality score. (lower == better) */ + /* Otherwise pick the best upstream from among roots and any other + * designated upstreams that we trust. */ unsigned int bestQualityOverall = ~((unsigned int)0); unsigned int bestQualityNotAvoid = ~((unsigned int)0); @@ -189,25 +190,34 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi const SharedPtr *bestNotAvoid = (const SharedPtr *)0; for(std::vector
::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { - const SharedPtr *const p = _peers.get(*a); - if (p) { - bool avoiding = false; - for(unsigned int i=0;iaddress()) { - avoiding = true; - break; - } - } - const unsigned int q = (*p)->relayQuality(now); - if (q <= bestQualityOverall) { - bestQualityOverall = q; - bestOverall = &(*p); + const SharedPtr *p = _peers.get(*a); + + if (!p) { + const Identity id(_getIdentity(*a)); + if (id) { + p = &(_peers.set(*a,SharedPtr(new Peer(RR,RR->identity,id)))); + } else { + RR->sw->requestWhois(*a); } - if ((!avoiding)&&(q <= bestQualityNotAvoid)) { - bestQualityNotAvoid = q; - bestNotAvoid = &(*p); + continue; // always skip since even if we loaded it, it's not going to be ready + } + + bool avoiding = false; + for(unsigned int i=0;iaddress()) { + avoiding = true; + break; } } + const unsigned int q = (*p)->relayQuality(now); + if (q <= bestQualityOverall) { + bestQualityOverall = q; + bestOverall = &(*p); + } + if ((!avoiding)&&(q <= bestQualityNotAvoid)) { + bestQualityNotAvoid = q; + bestNotAvoid = &(*p); + } } if (bestNotAvoid) { @@ -238,8 +248,19 @@ void Topology::setUpstream(const Address &a,bool upstream) Mutex::Lock _l(_lock); if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) { if (upstream) { - if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) + if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) { _upstreamAddresses.push_back(a); + + const SharedPtr *p = _peers.get(a); + if (!p) { + const Identity id(_getIdentity(a)); + if (id) { + _peers.set(a,SharedPtr(new Peer(RR,RR->identity,id))); + } else { + RR->sw->requestWhois(a); + } + } + } } else { std::vector
ua; for(std::vector
::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) { @@ -326,10 +347,8 @@ void Topology::_setWorld(const World &newWorld) _amRoot = true; } else { SharedPtr *rp = _peers.get(r->identity.address()); - if (!rp) { - SharedPtr newrp(new Peer(RR,RR->identity,r->identity)); - _peers.set(r->identity.address(),newrp); - } + if (!rp) + _peers.set(r->identity.address(),SharedPtr(new Peer(RR,RR->identity,r->identity))); } } -- cgit v1.2.3 From ab4021dd0ee37af0af4137dc772911ea8ec52bb2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 11:09:19 -0800 Subject: Do packet MAC check before locallyValidate(), and add timing measurement in selftest. --- node/IncomingPacket.cpp | 15 ++++++++------- selftest.cpp | 12 ++++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c6346346..ee4d62c0 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -275,7 +275,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut // Continue at // VALID } - } // else continue at // VALID + } // else if alreadyAuthenticated then continue at // VALID } else { // We don't already have an identity with this address -- validate and learn it @@ -285,18 +285,19 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } + // Check packet integrity and MAC + SharedPtr newPeer(new Peer(RR,RR->identity,id)); + if (!dearmor(newPeer->key())) { + TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; + } + // Check that identity's address is valid as per the derivation function if (!id.locallyValidate()) { TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); return true; } - // Check packet integrity and authentication - SharedPtr newPeer(new Peer(RR,RR->identity,id)); - if (!dearmor(newPeer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); - return true; - } peer = RR->topology->addPeer(newPeer); // Continue at // VALID diff --git a/selftest.cpp b/selftest.cpp index 7ca4ac3b..9992d757 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -376,11 +376,15 @@ static int testIdentity() std::cout << "FAIL (1)" << std::endl; return -1; } - if (!id.locallyValidate()) { - std::cout << "FAIL (2)" << std::endl; - return -1; + const uint64_t vst = OSUtils::now(); + for(int k=0;k<10;++k) { + if (!id.locallyValidate()) { + std::cout << "FAIL (2)" << std::endl; + return -1; + } } - std::cout << "PASS" << std::endl; + const uint64_t vet = OSUtils::now(); + std::cout << "PASS (" << ((double)(vet - vst) / 10.0) << "ms per validation)" << std::endl; std::cout << "[identity] Validate known-bad identity... "; std::cout.flush(); if (!id.fromString(KNOWN_BAD_IDENTITY)) { -- cgit v1.2.3 From 2ea9f516e121ea6eb344a8d180a739a1d707aecb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 12:59:04 -0800 Subject: Rate gate expensive validation of new identities in HELLO. --- node/Constants.hpp | 20 ++++++++++++++++++++ node/IncomingPacket.cpp | 10 +++++++++- node/InetAddress.hpp | 24 ++++++++++++++++++++++++ node/Node.cpp | 1 + node/Node.hpp | 22 ++++++++++++++++++++++ selftest.cpp | 11 +++++++++++ 6 files changed, 87 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 6400e289..8803ecee 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -375,6 +375,26 @@ */ #define ZT_PEER_GENERAL_RATE_LIMIT 1000 +/** + * Don't do expensive identity validation more often than this + * + * IPv4 and IPv6 address prefixes are hashed down to 14-bit (0-16383) integers + * using the first 24 bits for IPv4 or the first 48 bits for IPv6. These are + * then rate limited to one identity validation per this often milliseconds. + */ +#if (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64) || defined(_M_AMD64)) +// AMD64 machines can do anywhere from one every 50ms to one every 10ms. This provides plenty of margin. +#define ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT 2000 +#else +#if (defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__I86__)) +// 32-bit Intel machines usually average about one every 100ms +#define ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT 5000 +#else +// This provides a safe margin for ARM, MIPS, etc. that usually average one every 250-400ms +#define ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT 10000 +#endif +#endif + /** * How long is a path or peer considered to have a trust relationship with us (for e.g. relay policy) since last trusted established packet? */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ee4d62c0..41f3e47d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -247,6 +247,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut if (peer->identity() != id) { // Identity is different from the one we already have -- address collision + // Check rate limits + if (!RR->node->rateGateIdentityVerification(now,_path->address())) + return true; + uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { if (dearmor(key)) { // ensure packet is authentic, otherwise drop @@ -285,7 +289,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } - // Check packet integrity and MAC + // Check rate limits + if (!RR->node->rateGateIdentityVerification(now,_path->address())) + return true; + + // Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap) SharedPtr newPeer(new Peer(RR,RR->identity,id)); if (!dearmor(newPeer->key())) { TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 6f070fbf..1dff710d 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -449,6 +449,30 @@ struct InetAddress : public sockaddr_storage bool isNetwork() const throw(); + /** + * @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP + */ + inline unsigned long rateGateHash() const + { + unsigned long h = 0; + switch(ss_family) { + case AF_INET: + h = (Utils::ntoh((uint32_t)reinterpret_cast(this)->sin_addr.s_addr) & 0xffffff00) >> 8; + h ^= (h >> 14); + break; + case AF_INET6: { + const uint8_t *ip = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); + h = ((unsigned long)ip[0]); h <<= 1; + h += ((unsigned long)ip[1]); h <<= 1; + h += ((unsigned long)ip[2]); h <<= 1; + h += ((unsigned long)ip[3]); h <<= 1; + h += ((unsigned long)ip[4]); h <<= 1; + h += ((unsigned long)ip[5]); + } break; + } + return (h & 0x3fff); + } + /** * @return True if address family is non-zero */ diff --git a/node/Node.cpp b/node/Node.cpp index add3117e..ec719668 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -78,6 +78,7 @@ Node::Node( memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); + memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); // Use Salsa20 alone as a high-quality non-crypto PRNG { diff --git a/node/Node.hpp b/node/Node.hpp index e616da3d..ee0d6c4c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -283,6 +283,24 @@ public: return false; } + /** + * Check whether we should do potentially expensive identity verification (rate limit) + * + * @param now Current time + * @param from Source address of packet + * @return True if within rate limits + */ + inline bool rateGateIdentityVerification(const uint64_t now,const InetAddress &from) + { + unsigned long iph = from.rateGateHash(); + printf("%s %.4lx\n",from.toString().c_str(),iph); + if ((now - _lastIdentityVerification[iph]) >= ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT) { + _lastIdentityVerification[iph] = now; + return true; + } + return false; + } + virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig); virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); @@ -302,9 +320,13 @@ private: void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P + // For tracking packet IDs to filter out OK/ERROR replies to packets we did not send uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1]; uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; + // Time of last identity verification indexed by InetAddress.rateGateHash() + uint64_t _lastIdentityVerification[16384]; + ZT_DataStoreGetFunction _dataStoreGetFunction; ZT_DataStorePutFunction _dataStorePutFunction; ZT_WirePacketSendFunction _wirePacketSendFunction; diff --git a/selftest.cpp b/selftest.cpp index 9992d757..adac2f58 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -327,6 +327,17 @@ static int testCrypto() } std::cout << "PASS" << std::endl; + std::cout << "[crypto] Benchmarking C25519 ECC key agreement... "; std::cout.flush(); + C25519::Pair bp[8]; + for(int k=0;k<8;++k) + bp[k] = C25519::generate(); + const uint64_t st = OSUtils::now(); + for(unsigned int k=0;k<50;++k) { + C25519::agree(bp[~k & 7],bp[k & 7].pub,buf1,64); + } + const uint64_t et = OSUtils::now(); + std::cout << ((double)(et - st) / 50.0) << "ms per agreement." << std::endl; + std::cout << "[crypto] Testing Ed25519 ECC signatures... "; std::cout.flush(); C25519::Pair didntSign = C25519::generate(); for(unsigned int i=0;i<10;++i) { -- cgit v1.2.3 From 25f9c294dc677576ded51025b2c7e6397bdc11c0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 13:01:45 -0800 Subject: Small bug fix and warning removal. --- controller/EmbeddedNetworkController.cpp | 12 +++++++----- node/InetAddress.hpp | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b78f847e..74937dd8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1776,11 +1776,13 @@ void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,con std::map,uint64_t>::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); } - Dictionary *metaData = new Dictionary(mdstr.c_str()); - try { - this->request(nwid,InetAddress(),0,id,*metaData); - } catch ( ... ) {} - delete metaData; + if (online) { + Dictionary *metaData = new Dictionary(mdstr.c_str()); + try { + this->request(nwid,InetAddress(),0,id,*metaData); + } catch ( ... ) {} + delete metaData; + } } } catch ( ... ) {} } diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 1dff710d..c37fa621 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -427,7 +427,7 @@ struct InetAddress : public sockaddr_storage } else { unsigned long tmp = reinterpret_cast(this)->sin6_port; const uint8_t *a = reinterpret_cast(this); - for(long i=0;i(&tmp)[i % sizeof(tmp)] ^= a[i]; return tmp; } -- cgit v1.2.3 From 6e1da35c121c9a01ee4a487462660a5d7d3503a2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 13:15:58 -0800 Subject: Remove debug. --- node/Node.hpp | 1 - 1 file changed, 1 deletion(-) (limited to 'node') diff --git a/node/Node.hpp b/node/Node.hpp index ee0d6c4c..ba657691 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -293,7 +293,6 @@ public: inline bool rateGateIdentityVerification(const uint64_t now,const InetAddress &from) { unsigned long iph = from.rateGateHash(); - printf("%s %.4lx\n",from.toString().c_str(),iph); if ((now - _lastIdentityVerification[iph]) >= ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT) { _lastIdentityVerification[iph] = now; return true; -- cgit v1.2.3 From 673c0c811ea443c217b3a4ca17eeaed3ab596501 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 13:48:49 -0800 Subject: Wire through upstream stuff and add setRole(). --- include/ZeroTierOne.h | 11 +++++++++++ node/Node.cpp | 14 +++++++++++++- node/Node.hpp | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index d0fef1f1..67232cd2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1784,6 +1784,17 @@ int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage */ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node); +/** + * Set peer role + * + * Right now this can only be used to set a peer to either LEAF or + * UPSTREAM, since roots are fixed and defined by the World. + * + * @param ztAddress ZeroTier address (least significant 40 bits) + * @param role New peer role (LEAF or UPSTREAM) + */ +void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role); + /** * Set a network configuration master instance for this node * diff --git a/node/Node.cpp b/node/Node.cpp index ec719668..3d15f5bc 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -412,7 +412,7 @@ ZT_PeerList *Node::peers() const p->versionRev = -1; } p->latency = pi->second->latency(); - p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF; + p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : (RR->topology->isUpstream(pi->second->identity()) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF); std::vector< std::pair< SharedPtr,bool > > paths(pi->second->paths(_now)); SharedPtr bestp(pi->second->getBestPath(_now,false)); @@ -484,6 +484,11 @@ void Node::clearLocalInterfaceAddresses() _directPaths.clear(); } +void Node::setRole(uint64_t ztAddress,ZT_PeerRole role) +{ + RR->topology->setUpstream(Address(ztAddress),(role == ZT_PEER_ROLE_UPSTREAM)); +} + void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); @@ -1007,6 +1012,13 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node) } catch ( ... ) {} } +void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role) +{ + try { + reinterpret_cast(node)->setRole(ztAddress,role); + } catch ( ... ) {} +} + void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) { try { diff --git a/node/Node.hpp b/node/Node.hpp index ba657691..38303f8c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -105,6 +105,7 @@ public: void freeQueryResult(void *qr); int addLocalInterfaceAddress(const struct sockaddr_storage *addr); void clearLocalInterfaceAddresses(); + void setRole(uint64_t ztAddress,ZT_PeerRole role); void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); -- cgit v1.2.3 From ccdd4ffda70a35219fae1cdce1306f3d4ebd8dc9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Nov 2016 15:49:28 -0800 Subject: Move split() to OSUtils since it is not used in core. --- controller/EmbeddedNetworkController.cpp | 2 +- controller/JSONDB.cpp | 2 +- node/Utils.cpp | 44 -------------------------------- node/Utils.hpp | 11 -------- one.cpp | 4 +-- osdep/OSUtils.cpp | 44 ++++++++++++++++++++++++++++++++ osdep/OSUtils.hpp | 11 ++++++++ service/ControlPlane.cpp | 4 +-- 8 files changed, 61 insertions(+), 61 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 74937dd8..974936e4 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -889,7 +889,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &v6m = b["v6AssignMode"]; if (!nv6m.is_object()) nv6m = json::object(); if (v6m.is_string()) { // backward compatibility - std::vector v6ms(Utils::split(_jS(v6m,"").c_str(),",","","")); + std::vector v6ms(OSUtils::split(_jS(v6m,"").c_str(),",","","")); std::sort(v6ms.begin(),v6ms.end()); v6ms.erase(std::unique(v6ms.begin(),v6ms.end()),v6ms.end()); nv6m["rfc4193"] = false; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 6adc8a66..a9e9d05b 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -144,7 +144,7 @@ bool JSONDB::_isValidObjectName(const std::string &n) std::string JSONDB::_genPath(const std::string &n,bool create) { - std::vector pt(Utils::split(n.c_str(),"/","","")); + std::vector pt(OSUtils::split(n.c_str(),"/","","")); if (pt.size() == 0) return std::string(); diff --git a/node/Utils.cpp b/node/Utils.cpp index 9d67fc22..215e3b3e 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -218,50 +218,6 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) s20.encrypt12(buf,buf,bytes); } -std::vector Utils::split(const char *s,const char *const sep,const char *esc,const char *quot) -{ - std::vector fields; - std::string buf; - - if (!esc) - esc = ""; - if (!quot) - quot = ""; - - bool escapeState = false; - char quoteState = 0; - while (*s) { - if (escapeState) { - escapeState = false; - buf.push_back(*s); - } else if (quoteState) { - if (*s == quoteState) { - quoteState = 0; - fields.push_back(buf); - buf.clear(); - } else buf.push_back(*s); - } else { - const char *quotTmp; - if (strchr(esc,*s)) - escapeState = true; - else if ((buf.size() <= 0)&&((quotTmp = strchr(quot,*s)))) - quoteState = *quotTmp; - else if (strchr(sep,*s)) { - if (buf.size() > 0) { - fields.push_back(buf); - buf.clear(); - } // else skip runs of seperators - } else buf.push_back(*s); - } - ++s; - } - - if (buf.size()) - fields.push_back(buf); - - return fields; -} - bool Utils::scopy(char *dest,unsigned int len,const char *src) { if (!len) diff --git a/node/Utils.hpp b/node/Utils.hpp index cfe56501..48c43da3 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -111,17 +111,6 @@ public: */ static void getSecureRandom(void *buf,unsigned int bytes); - /** - * Split a string by delimiter, with optional escape and quote characters - * - * @param s String to split - * @param sep One or more separators - * @param esc Zero or more escape characters - * @param quot Zero or more quote characters - * @return Vector of tokens - */ - static std::vector split(const char *s,const char *const sep,const char *esc,const char *quot); - /** * Tokenize a string (alias for strtok_r or strtok_s depending on platform) * diff --git a/one.cpp b/one.cpp index 51cda0c7..3dad2ed9 100644 --- a/one.cpp +++ b/one.cpp @@ -696,7 +696,7 @@ static int idtool(int argc,char **argv) CertificateOfMembership com; for(int a=3;a params(Utils::split(argv[a],",","","")); + std::vector params(OSUtils::split(argv[a],",","","")); if (params.size() == 3) { uint64_t qId = Utils::hexStrToU64(params[0].c_str()); uint64_t qValue = Utils::hexStrToU64(params[1].c_str()); @@ -1084,7 +1084,7 @@ int main(int argc,char **argv) fprintf(stderr,"%s: no home path specified and no platform default available" ZT_EOL_S,argv[0]); return 1; } else { - std::vector hpsp(Utils::split(homeDir.c_str(),ZT_PATH_SEPARATOR_S,"","")); + std::vector hpsp(OSUtils::split(homeDir.c_str(),ZT_PATH_SEPARATOR_S,"","")); std::string ptmp; if (homeDir[0] == ZT_PATH_SEPARATOR) ptmp.push_back(ZT_PATH_SEPARATOR); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 4a81625b..65704fa3 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -315,6 +315,50 @@ bool OSUtils::writeFile(const char *path,const void *buf,unsigned int len) return false; } +std::vector OSUtils::split(const char *s,const char *const sep,const char *esc,const char *quot) +{ + std::vector fields; + std::string buf; + + if (!esc) + esc = ""; + if (!quot) + quot = ""; + + bool escapeState = false; + char quoteState = 0; + while (*s) { + if (escapeState) { + escapeState = false; + buf.push_back(*s); + } else if (quoteState) { + if (*s == quoteState) { + quoteState = 0; + fields.push_back(buf); + buf.clear(); + } else buf.push_back(*s); + } else { + const char *quotTmp; + if (strchr(esc,*s)) + escapeState = true; + else if ((buf.size() <= 0)&&((quotTmp = strchr(quot,*s)))) + quoteState = *quotTmp; + else if (strchr(sep,*s)) { + if (buf.size() > 0) { + fields.push_back(buf); + buf.clear(); + } // else skip runs of seperators + } else buf.push_back(*s); + } + ++s; + } + + if (buf.size()) + fields.push_back(buf); + + return fields; +} + std::string OSUtils::platformDefaultHomePath() { #ifdef __UNIX_LIKE__ diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index c481780b..c73f0bd3 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -236,6 +236,17 @@ public: */ static bool writeFile(const char *path,const void *buf,unsigned int len); + /** + * Split a string by delimiter, with optional escape and quote characters + * + * @param s String to split + * @param sep One or more separators + * @param esc Zero or more escape characters + * @param quot Zero or more quote characters + * @return Vector of tokens + */ + static std::vector split(const char *s,const char *const sep,const char *esc,const char *quot); + /** * Write a block of data to disk, replacing any current file contents * diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 150bba1b..07304fc9 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -266,7 +266,7 @@ unsigned int ControlPlane::handleRequest( { char json[8194]; unsigned int scode = 404; - std::vector ps(Utils::split(path.c_str(),"/","","")); + std::vector ps(OSUtils::split(path.c_str(),"/","","")); std::map urlArgs; Mutex::Lock _l(_lock); @@ -279,7 +279,7 @@ unsigned int ControlPlane::handleRequest( if (qpos != std::string::npos) { std::string args(ps[ps.size() - 1].substr(qpos + 1)); ps[ps.size() - 1] = ps[ps.size() - 1].substr(0,qpos); - std::vector asplit(Utils::split(args.c_str(),"&","","")); + std::vector asplit(OSUtils::split(args.c_str(),"&","","")); for(std::vector::iterator a(asplit.begin());a!=asplit.end();++a) { std::size_t eqpos = a->find('='); if (eqpos == std::string::npos) -- cgit v1.2.3 From 97d915b06c52d4bf03fd327a4365991262802475 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 21 Nov 2016 15:35:18 -0800 Subject: Expose relay policy in node settings. --- include/ZeroTierOne.h | 22 +++++++++++++++------- node/Node.cpp | 1 + service/ControlPlane.cpp | 2 ++ 3 files changed, 18 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 67232cd2..399f090c 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -396,6 +396,16 @@ enum ZT_Event ZT_EVENT_TRACE = 5 }; +/** + * Node relay policy + */ +enum ZT_RelayPolicy +{ + ZT_RELAY_POLICY_NEVER = 0, + ZT_RELAY_POLICY_TRUSTED = 1, + ZT_RELAY_POLICY_ALWAYS = 2 +}; + /** * Current node status */ @@ -430,6 +440,11 @@ typedef struct */ const char *secretIdentity; + /** + * Node relay policy + */ + ZT_RelayPolicy relayPolicy; + /** * True if some kind of connectivity appears available */ @@ -791,13 +806,6 @@ enum ZT_VirtualNetworkConfigOperation ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4 }; -enum ZT_RelayPolicy -{ - ZT_RELAY_POLICY_NEVER = 0, - ZT_RELAY_POLICY_TRUSTED = 1, - ZT_RELAY_POLICY_ALWAYS = 2 -}; - /** * What trust hierarchy role does this peer have? */ diff --git a/node/Node.cpp b/node/Node.cpp index 3d15f5bc..9ff7f197 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -384,6 +384,7 @@ void Node::status(ZT_NodeStatus *status) const status->worldTimestamp = RR->topology->worldTimestamp(); status->publicIdentity = RR->publicIdentityStr.c_str(); status->secretIdentity = RR->secretIdentityStr.c_str(); + status->relayPolicy = _relayPolicy; status->online = _online ? 1 : 0; } diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 07304fc9..2f9e746e 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -389,6 +389,7 @@ unsigned int ControlPlane::handleRequest( "\t\"worldId\": %llu,\n" "\t\"worldTimestamp\": %llu,\n" "\t\"online\": %s,\n" + "\t\"relayPolicy\": \"%s\",\n" "\t\"tcpFallbackActive\": %s,\n" "\t\"versionMajor\": %d,\n" "\t\"versionMinor\": %d,\n" @@ -402,6 +403,7 @@ unsigned int ControlPlane::handleRequest( status.worldId, status.worldTimestamp, (status.online) ? "true" : "false", + ((status.relayPolicy == ZT_RELAY_POLICY_ALWAYS) ? "always" : ((status.relayPolicy == ZT_RELAY_POLICY_NEVER) ? "never" : "trusted")), (_svc->tcpFallbackActive()) ? "true" : "false", ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, -- cgit v1.2.3 From cbaef66e82eeec05dfb005bd34cafc0c1cb411f7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 21 Nov 2016 16:04:01 -0800 Subject: Fix a deadlock in federation/upstream code. --- node/Node.cpp | 48 ++++++++++++++++++-------------- node/Topology.cpp | 83 ++++++++++++++++++++++++++----------------------------- 2 files changed, 66 insertions(+), 65 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 9ff7f197..263cfc6e 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -174,12 +174,14 @@ ZT_ResultCode Node::processVirtualNetworkFrame( } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } +// Closure used to ping upstream and active/online peers class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,const std::vector
&upstreams,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), + _upstreams(upstreams), _now(now), _world(RR->topology->world()) { @@ -189,29 +191,25 @@ public: inline void operator()(Topology &t,const SharedPtr &p) { - bool upstream = false; - InetAddress stableEndpoint4,stableEndpoint6; - - // If this is a world root, pick (if possible) both an IPv4 and an IPv6 stable endpoint to use if link isn't currently alive. - for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { - if (r->identity == p->identity()) { - upstream = true; - for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)r->stableEndpoints.size();++k) { - const InetAddress &addr = r->stableEndpoints[ptr++ % r->stableEndpoints.size()]; - if (!stableEndpoint4) { - if (addr.ss_family == AF_INET) - stableEndpoint4 = addr; - } - if (!stableEndpoint6) { - if (addr.ss_family == AF_INET6) - stableEndpoint6 = addr; + if (std::find(_upstreams.begin(),_upstreams.end(),p->address()) != _upstreams.end()) { + InetAddress stableEndpoint4,stableEndpoint6; + for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { + if (r->identity == p->identity()) { + for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)r->stableEndpoints.size();++k) { + const InetAddress &addr = r->stableEndpoints[ptr++ % r->stableEndpoints.size()]; + if (!stableEndpoint4) { + if (addr.ss_family == AF_INET) + stableEndpoint4 = addr; + } + if (!stableEndpoint6) { + if (addr.ss_family == AF_INET6) + stableEndpoint6 = addr; + } } + break; } - break; } - } - if (upstream) { // We keep connections to upstream peers alive forever. bool needToContactIndirect = true; if (p->doPingAndKeepalive(_now,AF_INET)) { @@ -246,6 +244,7 @@ public: private: const RuntimeEnvironment *RR; + const std::vector
&_upstreams; uint64_t _now; World _world; }; @@ -274,8 +273,15 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) (*n)->requestConfiguration(); + // Run WHOIS on upstreams we don't know about + const std::vector
upstreams(RR->topology->upstreamAddresses()); + for(std::vector
::const_iterator a(upstreams.begin());a!=upstreams.end();++a) { + if (!RR->topology->getPeer(*a)) + RR->sw->requestWhois(*a); + } + // Do pings and keepalives - _PingPeersThatNeedPing pfunc(RR,now); + _PingPeersThatNeedPing pfunc(RR,upstreams,now); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); // Update online status, post status change as event diff --git a/node/Topology.cpp b/node/Topology.cpp index 81382e05..5aafc439 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -191,32 +191,23 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi for(std::vector
::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { const SharedPtr *p = _peers.get(*a); - - if (!p) { - const Identity id(_getIdentity(*a)); - if (id) { - p = &(_peers.set(*a,SharedPtr(new Peer(RR,RR->identity,id)))); - } else { - RR->sw->requestWhois(*a); + if (p) { + bool avoiding = false; + for(unsigned int i=0;iaddress()) { + avoiding = true; + break; + } } - continue; // always skip since even if we loaded it, it's not going to be ready - } - - bool avoiding = false; - for(unsigned int i=0;iaddress()) { - avoiding = true; - break; + const unsigned int q = (*p)->relayQuality(now); + if (q <= bestQualityOverall) { + bestQualityOverall = q; + bestOverall = &(*p); + } + if ((!avoiding)&&(q <= bestQualityNotAvoid)) { + bestQualityNotAvoid = q; + bestNotAvoid = &(*p); } - } - const unsigned int q = (*p)->relayQuality(now); - if (q <= bestQualityOverall) { - bestQualityOverall = q; - bestOverall = &(*p); - } - if ((!avoiding)&&(q <= bestQualityNotAvoid)) { - bestQualityNotAvoid = q; - bestNotAvoid = &(*p); } } @@ -245,31 +236,35 @@ bool Topology::isUpstream(const Identity &id) const void Topology::setUpstream(const Address &a,bool upstream) { - Mutex::Lock _l(_lock); - if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) { - if (upstream) { - if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) { - _upstreamAddresses.push_back(a); - - const SharedPtr *p = _peers.get(a); - if (!p) { - const Identity id(_getIdentity(a)); - if (id) { - _peers.set(a,SharedPtr(new Peer(RR,RR->identity,id))); - } else { - RR->sw->requestWhois(a); + bool needWhois = false; + { + Mutex::Lock _l(_lock); + if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) { + if (upstream) { + if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) { + _upstreamAddresses.push_back(a); + const SharedPtr *p = _peers.get(a); + if (!p) { + const Identity id(_getIdentity(a)); + if (id) { + _peers.set(a,SharedPtr(new Peer(RR,RR->identity,id))); + } else { + needWhois = true; // need to do this later due to _lock + } } } + } else { + std::vector
ua; + for(std::vector
::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) { + if (a != *i) + ua.push_back(*i); + } + _upstreamAddresses.swap(ua); } - } else { - std::vector
ua; - for(std::vector
::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) { - if (a != *i) - ua.push_back(*i); - } - _upstreamAddresses.swap(ua); } } + if (needWhois) + RR->sw->requestWhois(a); } bool Topology::worldUpdateIfValid(const World &newWorld) -- cgit v1.2.3 From 42ba70e79e3f1484f7bdde5832658cbd179649dc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 22 Nov 2016 10:54:58 -0800 Subject: Replace long callback arg list with struct, and implement path whitelisting, path blacklisting, and local.conf support for roles. --- include/ZeroTierOne.h | 97 ++++++++++++++++---- node/IncomingPacket.cpp | 6 +- node/Node.cpp | 88 ++++++------------ node/Node.hpp | 86 +++-------------- node/Peer.cpp | 2 +- node/Switch.cpp | 2 +- service/OneService.cpp | 238 ++++++++++++++++++++++++++++++++++++------------ 7 files changed, 302 insertions(+), 217 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 399f090c..72da53f2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1495,8 +1495,9 @@ typedef int (*ZT_WirePacketSendFunction)( * Paramters: * (1) Node * (2) User pointer - * (3) Local interface address - * (4) Remote address + * (3) ZeroTier address or 0 for none/any + * (4) Local interface address + * (5) Remote address * * This function must return nonzero (true) if the path should be used. * @@ -1515,13 +1516,87 @@ typedef int (*ZT_WirePacketSendFunction)( typedef int (*ZT_PathCheckFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + uint64_t, /* ZeroTier address */ const struct sockaddr_storage *, /* Local address */ const struct sockaddr_storage *); /* Remote address */ +/** + * Function to get physical addresses for ZeroTier peers + * + * Parameters: + * (1) Node + * (2) User pointer + * (3) ZeroTier address (least significant 40 bits) + * (4) Desried address family or -1 for any + * (5) Buffer to fill with result + * + * If provided this function will be occasionally called to get physical + * addresses that might be tried to reach a ZeroTier address. It must + * return a nonzero (true) value if the result buffer has been filled + * with an address. + */ +typedef int (*ZT_PathLookupFunction)( + ZT_Node *, /* Node */ + void *, /* User ptr */ + uint64_t, /* ZeroTier address (40 bits) */ + int, /* Desired ss_family or -1 for any */ + struct sockaddr_storage *); /* Result buffer */ + /****************************************************************************/ /* C Node API */ /****************************************************************************/ +/** + * Structure for configuring ZeroTier core callback functions + */ +struct ZT_Node_Callbacks +{ + /** + * Struct version -- must currently be 0 + */ + long version; + + /** + * REQUIRED: Function to get objects from persistent storage + */ + ZT_DataStoreGetFunction dataStoreGetFunction; + + /** + * REQUIRED: Function to store objects in persistent storage + */ + ZT_DataStorePutFunction dataStorePutFunction; + + /** + * REQUIRED: Function to send packets over the physical wire + */ + ZT_WirePacketSendFunction wirePacketSendFunction; + + /** + * REQUIRED: Function to inject frames into a virtual network's TAP + */ + ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction; + + /** + * REQUIRED: Function to be called when virtual networks are configured or changed + */ + ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction; + + /** + * REQUIRED: Function to be called to notify external code of important events + */ + ZT_EventCallback eventCallback; + + /** + * OPTIONAL: Function to check whether a given physical path should be used + */ + ZT_PathCheckFunction pathCheckFunction; + + /** + * OPTIONAL: Function to get hints to physical paths to ZeroTier addresses + */ + ZT_PathLookupFunction pathLookupFunction; +}; + /** * Create a new ZeroTier One node * @@ -1533,25 +1608,11 @@ typedef int (*ZT_PathCheckFunction)( * * @param node Result: pointer is set to new node instance on success * @param uptr User pointer to pass to functions/callbacks + * @param callbacks Callback function configuration * @param now Current clock in milliseconds - * @param dataStoreGetFunction Function called to get objects from persistent storage - * @param dataStorePutFunction Function called to put objects in persistent storage - * @param virtualNetworkConfigFunction Function to be called when virtual LANs are created, deleted, or their config parameters change - * @param pathCheckFunction A function to check whether a path should be used for ZeroTier traffic, or NULL to allow any path - * @param eventCallback Function to receive status updates and non-fatal error notices * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_new( - ZT_Node **node, - void *uptr, - uint64_t now, - ZT_DataStoreGetFunction dataStoreGetFunction, - ZT_DataStorePutFunction dataStorePutFunction, - ZT_WirePacketSendFunction wirePacketSendFunction, - ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction, - ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction, - ZT_PathCheckFunction pathCheckFunction, - ZT_EventCallback eventCallback); +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); /** * Delete a node and free all resources it consumes diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 41f3e47d..7b828f8b 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -552,7 +552,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) { + if (RR->node->shouldUsePathForZeroTierTraffic(with,_path->localAddress(),atAddr)) { RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -1120,7 +1120,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(InetAddress(),a,now); @@ -1139,7 +1139,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(InetAddress(),a,now); diff --git a/node/Node.cpp b/node/Node.cpp index 263cfc6e..a180766b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -46,34 +46,20 @@ namespace ZeroTier { /* Public Node interface (C++, exposed via CAPI bindings) */ /****************************************************************************/ -Node::Node( - uint64_t now, - void *uptr, - ZT_DataStoreGetFunction dataStoreGetFunction, - ZT_DataStorePutFunction dataStorePutFunction, - ZT_WirePacketSendFunction wirePacketSendFunction, - ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction, - ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction, - ZT_PathCheckFunction pathCheckFunction, - ZT_EventCallback eventCallback) : +Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : _RR(this), RR(&_RR), _uPtr(uptr), - _dataStoreGetFunction(dataStoreGetFunction), - _dataStorePutFunction(dataStorePutFunction), - _wirePacketSendFunction(wirePacketSendFunction), - _virtualNetworkFrameFunction(virtualNetworkFrameFunction), - _virtualNetworkConfigFunction(virtualNetworkConfigFunction), - _pathCheckFunction(pathCheckFunction), - _eventCallback(eventCallback), - _networks(), - _networks_m(), _prngStreamPtr(0), _now(now), _lastPingCheck(0), _lastHousekeepingRun(0), _relayPolicy(ZT_RELAY_POLICY_TRUSTED) { + if (callbacks->version != 0) + throw std::runtime_error("callbacks struct version mismatch"); + memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); + _online = false; memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); @@ -81,30 +67,26 @@ Node::Node( memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); // Use Salsa20 alone as a high-quality non-crypto PRNG - { - char foo[32]; - Utils::getSecureRandom(foo,32); - _prng.init(foo,256,foo); - memset(_prngStream,0,sizeof(_prngStream)); - _prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream)); + char foo[32]; + Utils::getSecureRandom(foo,32); + _prng.init(foo,256,foo); + memset(_prngStream,0,sizeof(_prngStream)); + _prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream)); + + std::string idtmp(dataStoreGet("identity.secret")); + if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { + TRACE("identity.secret not found, generating..."); + RR->identity.generate(); + idtmp = RR->identity.toString(true); + if (!dataStorePut("identity.secret",idtmp,true)) + throw std::runtime_error("unable to write identity.secret"); } - - { - std::string idtmp(dataStoreGet("identity.secret")); - if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { - TRACE("identity.secret not found, generating..."); - RR->identity.generate(); - idtmp = RR->identity.toString(true); - if (!dataStorePut("identity.secret",idtmp,true)) - throw std::runtime_error("unable to write identity.secret"); - } - RR->publicIdentityStr = RR->identity.toString(false); - RR->secretIdentityStr = RR->identity.toString(true); - idtmp = dataStoreGet("identity.public"); - if (idtmp != RR->publicIdentityStr) { - if (!dataStorePut("identity.public",RR->publicIdentityStr,false)) - throw std::runtime_error("unable to write identity.public"); - } + RR->publicIdentityStr = RR->identity.toString(false); + RR->secretIdentityStr = RR->identity.toString(true); + idtmp = dataStoreGet("identity.public"); + if (idtmp != RR->publicIdentityStr) { + if (!dataStorePut("identity.public",RR->publicIdentityStr,false)) + throw std::runtime_error("unable to write identity.public"); } try { @@ -638,7 +620,7 @@ std::string Node::dataStoreGet(const char *name) std::string r; unsigned long olen = 0; do { - long n = _dataStoreGetFunction(reinterpret_cast(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); + long n = _cb.dataStoreGetFunction(reinterpret_cast(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); if (n <= 0) return std::string(); r.append(buf,n); @@ -646,7 +628,7 @@ std::string Node::dataStoreGet(const char *name) return r; } -bool Node::shouldUsePathForZeroTierTraffic(const InetAddress &localAddress,const InetAddress &remoteAddress) +bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) { if (!Path::isAddressValidForPath(remoteAddress)) return false; @@ -663,9 +645,7 @@ bool Node::shouldUsePathForZeroTierTraffic(const InetAddress &localAddress,const } } - if (_pathCheckFunction) - return (_pathCheckFunction(reinterpret_cast(this),_uPtr,reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0); - else return true; + return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0) : true); } #ifdef ZT_TRACE @@ -822,21 +802,11 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des extern "C" { -enum ZT_ResultCode ZT_Node_new( - ZT_Node **node, - void *uptr, - uint64_t now, - ZT_DataStoreGetFunction dataStoreGetFunction, - ZT_DataStorePutFunction dataStorePutFunction, - ZT_WirePacketSendFunction wirePacketSendFunction, - ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction, - ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction, - ZT_PathCheckFunction pathCheckFunction, - ZT_EventCallback eventCallback) +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) { *node = (ZT_Node *)0; try { - *node = reinterpret_cast(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,pathCheckFunction,eventCallback)); + *node = reinterpret_cast(new ZeroTier::Node(uptr,callbacks,now)); return ZT_RESULT_OK; } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; diff --git a/node/Node.hpp b/node/Node.hpp index 38303f8c..7d99ff09 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -59,17 +59,7 @@ namespace ZeroTier { class Node : public NetworkController::Sender { public: - Node( - uint64_t now, - void *uptr, - ZT_DataStoreGetFunction dataStoreGetFunction, - ZT_DataStorePutFunction dataStorePutFunction, - ZT_WirePacketSendFunction wirePacketSendFunction, - ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction, - ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction, - ZT_PathCheckFunction pathCheckFunction, - ZT_EventCallback eventCallback); - + Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); virtual ~Node(); // Public API Functions ---------------------------------------------------- @@ -127,24 +117,11 @@ public: // Internal functions ------------------------------------------------------ - /** - * @return Time as of last call to run() - */ inline uint64_t now() const throw() { return _now; } - /** - * Enqueue a ZeroTier message to be sent - * - * @param localAddress Local address - * @param addr Destination address - * @param data Packet data - * @param len Packet length - * @param ttl Desired TTL (default: 0 for unchanged/default TTL) - * @return True if packet appears to have been sent - */ inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { - return (_wirePacketSendFunction( + return (_cb.wirePacketSendFunction( reinterpret_cast(this), _uPtr, reinterpret_cast(&localAddress), @@ -154,21 +131,9 @@ public: ttl) == 0); } - /** - * Enqueue a frame to be injected into a tap device (port) - * - * @param nwid Network ID - * @param nuptr Network user ptr - * @param source Source MAC - * @param dest Destination MAC - * @param etherType 16-bit ethernet type - * @param vlanId VLAN ID or 0 if none - * @param data Frame data - * @param len Frame length - */ inline void putFrame(uint64_t nwid,void **nuptr,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { - _virtualNetworkFrameFunction( + _cb.virtualNetworkFrameFunction( reinterpret_cast(this), _uPtr, nwid, @@ -181,13 +146,6 @@ public: len); } - /** - * @param localAddress Local address - * @param remoteAddress Remote address - * @return True if path should be used - */ - bool shouldUsePathForZeroTierTraffic(const InetAddress &localAddress,const InetAddress &remoteAddress); - inline SharedPtr network(uint64_t nwid) const { Mutex::Lock _l(_networks_m); @@ -214,37 +172,20 @@ public: return nw; } - /** - * @return Potential direct paths to me a.k.a. local interface addresses - */ inline std::vector directPaths() const { Mutex::Lock _l(_directPaths_m); return _directPaths; } - inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_dataStorePutFunction(reinterpret_cast(this),_uPtr,name,data,len,(int)secure) == 0); } + inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,name,data,len,(int)secure) == 0); } inline bool dataStorePut(const char *name,const std::string &data,bool secure) { return dataStorePut(name,(const void *)data.data(),(unsigned int)data.length(),secure); } - inline void dataStoreDelete(const char *name) { _dataStorePutFunction(reinterpret_cast(this),_uPtr,name,(const void *)0,0,0); } + inline void dataStoreDelete(const char *name) { _cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,name,(const void *)0,0,0); } std::string dataStoreGet(const char *name); - /** - * Post an event to the external user - * - * @param ev Event type - * @param md Meta-data (default: NULL/none) - */ - inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _eventCallback(reinterpret_cast(this),_uPtr,ev,md); } + inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast(this),_uPtr,ev,md); } - /** - * Update virtual network port configuration - * - * @param nwid Network ID - * @param nuptr Network user ptr - * @param op Configuration operation - * @param nc Network configuration - */ - inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,nwid,nuptr,op,nc); } + inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,nwid,nuptr,op,nc); } inline bool online() const throw() { return _online; } inline ZT_RelayPolicy relayPolicy() const { return _relayPolicy; } @@ -253,6 +194,9 @@ public: void postTrace(const char *module,unsigned int line,const char *fmt,...); #endif + bool shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); + inline bool getPathHint(const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } + uint64_t prng(); void postCircuitTestReport(const ZT_CircuitTestReport *report); void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); @@ -317,8 +261,8 @@ private: RuntimeEnvironment _RR; RuntimeEnvironment *RR; - void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P + ZT_Node_Callbacks _cb; // For tracking packet IDs to filter out OK/ERROR replies to packets we did not send uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1]; @@ -327,14 +271,6 @@ private: // Time of last identity verification indexed by InetAddress.rateGateHash() uint64_t _lastIdentityVerification[16384]; - ZT_DataStoreGetFunction _dataStoreGetFunction; - ZT_DataStorePutFunction _dataStorePutFunction; - ZT_WirePacketSendFunction _wirePacketSendFunction; - ZT_VirtualNetworkFrameFunction _virtualNetworkFrameFunction; - ZT_VirtualNetworkConfigFunction _virtualNetworkConfigFunction; - ZT_PathCheckFunction _pathCheckFunction; - ZT_EventCallback _eventCallback; - std::vector< std::pair< uint64_t, SharedPtr > > _networks; Mutex _networks_m; diff --git a/node/Peer.cpp b/node/Peer.cpp index 94fb5298..e0bd0eac 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -160,7 +160,7 @@ void Peer::received( } } - if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(path->localAddress(),path->address())) ) { + if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(_id.address(),path->localAddress(),path->address())) ) { if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); diff --git a/node/Switch.cpp b/node/Switch.cpp index a5dd57e4..881d7b92 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -85,7 +85,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from Address beaconAddr(reinterpret_cast(data) + 8,5); if (beaconAddr == RR->identity.address()) return; - if (!RR->node->shouldUsePathForZeroTierTraffic(localAddr,fromAddr)) + if (!RR->node->shouldUsePathForZeroTierTraffic(beaconAddr,localAddr,fromAddr)) return; SharedPtr peer(RR->topology->getPeer(beaconAddr)); if (peer) { // we'll only respond to beacons from known peers diff --git a/service/OneService.cpp b/service/OneService.cpp index efb6ff3c..7434ca67 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -160,7 +160,6 @@ static uint64_t _jI(const json &jv,const uint64_t dfl) } return dfl; } -/* static bool _jB(const json &jv,const bool dfl) { if (jv.is_boolean()) { @@ -181,7 +180,6 @@ static bool _jB(const json &jv,const bool dfl) } return dfl; } -*/ static std::string _jS(const json &jv,const char *dfl) { if (jv.is_string()) { @@ -452,7 +450,8 @@ static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name, static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure); static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); +static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); #ifdef ZT_ENABLE_CLUSTER static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len); @@ -536,11 +535,20 @@ public: const std::string _homePath; BackgroundResolver _tcpFallbackResolver; InetAddress _allowManagementFrom; - json _localConfig; EmbeddedNetworkController *_controller; Phy _phy; Node *_node; + // Local configuration and memo-ized static path definitions + json _localConfig; + Hashtable< uint64_t,std::vector > _v4Hints; + Hashtable< uint64_t,std::vector > _v6Hints; + Hashtable< uint64_t,std::vector > _v4Blacklists; + Hashtable< uint64_t,std::vector > _v6Blacklists; + std::vector< InetAddress > _globalV4Blacklist; + std::vector< InetAddress > _globalV6Blacklist; + Mutex _localConfig_m; + /* * To attempt to handle NAT/gateway craziness we use three local UDP ports: * @@ -552,7 +560,6 @@ public: * destructively with uPnP port mapping behavior in very weird buggy ways. * It's only used if uPnP/NAT-PMP is enabled in this build. */ - Binder _bindings[3]; unsigned int _ports[3]; uint16_t _portsBE[3]; // ports in big-endian network byte order as in sockaddr @@ -756,16 +763,19 @@ public: // Clean up any legacy files if present OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + "peers.save").c_str()); - _node = new Node( - OSUtils::now(), - this, - SnodeDataStoreGetFunction, - SnodeDataStorePutFunction, - SnodeWirePacketSendFunction, - SnodeVirtualNetworkFrameFunction, - SnodeVirtualNetworkConfigFunction, - SnodePathCheckFunction, - SnodeEventCallback); + { + struct ZT_Node_Callbacks cb; + cb.version = 0; + cb.dataStoreGetFunction = SnodeDataStoreGetFunction; + cb.dataStorePutFunction = SnodeDataStorePutFunction; + cb.wirePacketSendFunction = SnodeWirePacketSendFunction; + cb.virtualNetworkFrameFunction = SnodeVirtualNetworkFrameFunction; + cb.virtualNetworkConfigFunction = SnodeVirtualNetworkConfigFunction; + cb.eventCallback = SnodeEventCallback; + cb.pathCheckFunction = SnodePathCheckFunction; + cb.pathLookupFunction = SnodePathLookupFunction; + _node = new Node(this,&cb,OSUtils::now()); + } // Attempt to bind to a secondary port chosen from our ZeroTier address. // This exists because there are buggy NATs out there that fail if more @@ -842,6 +852,7 @@ public: } // Read local config file + Mutex::Lock _l2(_localConfig_m); std::string lcbuf; if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "local.conf").c_str(),lcbuf)) { try { @@ -854,19 +865,18 @@ public: } } - // Get any trusted paths in local.conf + // Get any trusted paths in local.conf (we'll parse the rest of physical[] elsewhere) json &physical = _localConfig["physical"]; if (physical.is_object()) { for(json::iterator phy(physical.begin());phy!=physical.end();++phy) { - std::string nstr = phy.key(); - if (nstr.length()) { + InetAddress net(_jS(phy.key(),"")); + if (net) { if (phy.value().is_object()) { - uint64_t tpid = 0; - if ((tpid = _jI(phy.value()["trustedPathId"],0ULL))) { - InetAddress trustedPathNetwork(nstr); - if ( ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (trustedPathNetwork.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (trustedPathNetwork.netmaskBits() > 0) ) { + uint64_t tpid; + if ((tpid = _jI(phy.value()["trustedPathId"],0ULL)) != 0ULL) { + if ( ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (net.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (net.netmaskBits() > 0) ) { trustedPathIds[trustedPathCount] = tpid; - trustedPathNetworks[trustedPathCount] = trustedPathNetwork; + trustedPathNetworks[trustedPathCount] = net; ++trustedPathCount; } } @@ -878,31 +888,8 @@ public: // Set trusted paths if there are any if (trustedPathCount) _node->setTrustedPaths(reinterpret_cast(trustedPathNetworks),trustedPathIds,trustedPathCount); - - // Set any roles (upstream/federation) - json &virt = _localConfig["virtual"]; - if (virt.is_object()) { - for(json::iterator v(virt.begin());v!=virt.end();++v) { - const std::string nstr = v.key(); - if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) { - const Address ztaddr(nstr.c_str()); - if (ztaddr) - _node->setRole(ztaddr.toInt(),(_jS(v.value()["role"],"") == "upstream") ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF); - } - } - } - - // Set any other local config stuff - json &settings = _localConfig["settings"]; - if (settings.is_object()) { - const std::string rp(_jS(settings["relayPolicy"],"")); - if (rp == "always") - _node->setRelayPolicy(ZT_RELAY_POLICY_ALWAYS); - else if (rp == "never") - _node->setRelayPolicy(ZT_RELAY_POLICY_NEVER); - else _node->setRelayPolicy(ZT_RELAY_POLICY_TRUSTED); - } } + applyLocalConfig(); _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str()); _node->setNetconfMaster((void *)_controller); @@ -1174,7 +1161,90 @@ public: return true; } - // Begin private implementation methods + // Internal implementation methods ----------------------------------------- + + void applyLocalConfig() + { + Mutex::Lock _l(_localConfig_m); + + _v4Hints.clear(); + _v6Hints.clear(); + _v4Blacklists.clear(); + _v6Blacklists.clear(); + json &virt = _localConfig["virtual"]; + if (virt.is_object()) { + for(json::iterator v(virt.begin());v!=virt.end();++v) { + const std::string nstr = v.key(); + if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) { + const Address ztaddr(nstr.c_str()); + if (ztaddr) { + _node->setRole(ztaddr.toInt(),(_jS(v.value()["role"],"") == "upstream") ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF); + + const uint64_t ztaddr2 = ztaddr.toInt(); + std::vector &v4h = _v4Hints[ztaddr2]; + std::vector &v6h = _v6Hints[ztaddr2]; + std::vector &v4b = _v4Blacklists[ztaddr2]; + std::vector &v6b = _v6Blacklists[ztaddr2]; + + json &tryAddrs = v.value()["try"]; + if (tryAddrs.is_array()) { + for(unsigned long i=0;i 0)) { + if (phy.value().is_object()) { + if (_jB(phy.value()["blacklist"],false)) { + if (net.ss_family == AF_INET) + _globalV4Blacklist.push_back(net); + else if (net.ss_family == AF_INET6) + _globalV6Blacklist.push_back(net); + } + } + } + } + } + + json &settings = _localConfig["settings"]; + if (settings.is_object()) { + const std::string rp(_jS(settings["relayPolicy"],"")); + if (rp == "always") + _node->setRelayPolicy(ZT_RELAY_POLICY_ALWAYS); + else if (rp == "never") + _node->setRelayPolicy(ZT_RELAY_POLICY_NEVER); + else _node->setRelayPolicy(ZT_RELAY_POLICY_TRUSTED); + } + } // Checks if a managed IP or route target is allowed bool checkIfManagedIsAllowed(const NetworkState &n,const InetAddress &target) @@ -1306,6 +1376,8 @@ public: } } + // Handlers for Node and Phy<> callbacks ----------------------------------- + inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) { #ifdef ZT_ENABLE_CLUSTER @@ -1783,21 +1855,48 @@ public: n->tap->put(MAC(sourceMac),MAC(destMac),etherType,data,len); } - inline int nodePathCheckFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) + inline int nodePathCheckFunction(uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) { - Mutex::Lock _l(_nets_m); - - for(std::map::const_iterator n(_nets.begin());n!=_nets.end();++n) { - if (n->second.tap) { - std::vector ips(n->second.tap->ips()); - for(std::vector::const_iterator i(ips.begin());i!=ips.end();++i) { - if (i->containsAddress(*(reinterpret_cast(remoteAddr)))) { - return 0; + // Make sure we're not trying to do ZeroTier-over-ZeroTier + { + Mutex::Lock _l(_nets_m); + for(std::map::const_iterator n(_nets.begin());n!=_nets.end();++n) { + if (n->second.tap) { + std::vector ips(n->second.tap->ips()); + for(std::vector::const_iterator i(ips.begin());i!=ips.end();++i) { + if (i->containsAddress(*(reinterpret_cast(remoteAddr)))) { + return 0; + } } } } } - + + // Check blacklists + const Hashtable< uint64_t,std::vector > *blh = (const Hashtable< uint64_t,std::vector > *)0; + const std::vector *gbl = (const std::vector *)0; + if (remoteAddr->ss_family == AF_INET) { + blh = &_v4Blacklists; + gbl = &_globalV4Blacklist; + } else if (remoteAddr->ss_family == AF_INET6) { + blh = &_v6Blacklists; + gbl = &_globalV6Blacklist; + } + if (blh) { + Mutex::Lock _l(_localConfig_m); + const std::vector *l = blh->get(ztaddr); + if (l) { + for(std::vector::const_iterator a(l->begin());a!=l->end();++a) { + if (a->containsAddress(*reinterpret_cast(remoteAddr))) + return 0; + } + } + for(std::vector::const_iterator a(gbl->begin());a!=gbl->end();++a) { + if (a->containsAddress(*reinterpret_cast(remoteAddr))) + return 0; + } + } + /* Note: I do not think we need to scan for overlap with managed routes * because of the "route forking" and interface binding that we do. This * ensures (we hope) that ZeroTier traffic will still take the physical @@ -1807,6 +1906,23 @@ public: return 1; } + inline int nodePathLookupFunction(uint64_t ztaddr,int family,struct sockaddr_storage *result) + { + const Hashtable< uint64_t,std::vector > *lh = (const Hashtable< uint64_t,std::vector > *)0; + if (family < 0) + lh = (_node->prng() & 1) ? &_v4Hints : &_v6Hints; + else if (family == AF_INET) + lh = &_v4Hints; + else if (family == AF_INET6) + lh = &_v6Hints; + else return 0; + const std::vector *l = lh->get(ztaddr); + if ((l)&&(l->size() > 0)) { + memcpy(result,&((*l)[(unsigned long)_node->prng() % l->size()]),sizeof(struct sockaddr_storage)); + return 1; + } else return 0; + } + inline void tapFrameHandler(uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { _node->processVirtualNetworkFrame(OSUtils::now(),nwid,from.toInt(),to.toInt(),etherType,vlanId,data,len,&_nextBackgroundTaskDeadline); @@ -1956,8 +2072,10 @@ static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct soc { return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); } static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); } -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) -{ return reinterpret_cast(uptr)->nodePathCheckFunction(localAddr,remoteAddr); } +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) +{ return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr,localAddr,remoteAddr); } +static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) +{ return reinterpret_cast(uptr)->nodePathLookupFunction(ztaddr,family,result); } #ifdef ZT_ENABLE_CLUSTER static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len) -- cgit v1.2.3 From 84732fcb12d66708e7887fba51413cbe629d86d3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 22 Nov 2016 14:23:13 -0800 Subject: Wire through external path lookup. Static paths should now work. --- node/Constants.hpp | 5 +++++ node/Node.cpp | 2 +- node/Node.hpp | 7 +++++-- node/Peer.cpp | 11 +++++++++++ node/Peer.hpp | 8 ++++++++ node/Switch.cpp | 15 ++++++++------- service/OneService.cpp | 13 +++++++------ 7 files changed, 45 insertions(+), 16 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 8803ecee..ac1919b3 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -320,6 +320,11 @@ */ #define ZT_MIN_UNITE_INTERVAL 30000 +/** + * How often should peers try memorized or statically defined paths? + */ +#define ZT_TRY_MEMORIZED_PATH_INTERVAL 30000 + /** * Sanity limit on maximum bridge routes * diff --git a/node/Node.cpp b/node/Node.cpp index a180766b..11f76365 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -683,7 +683,7 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) uint64_t Node::prng() { - unsigned int p = (++_prngStreamPtr % (sizeof(_prngStream) / sizeof(uint64_t))); + unsigned int p = (++_prngStreamPtr % ZT_NODE_PRNG_BUF_SIZE); if (!p) _prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream)); return _prngStream[p]; diff --git a/node/Node.hpp b/node/Node.hpp index 7d99ff09..eb46527d 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -49,6 +49,9 @@ #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 #define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 +// Size of PRNG stream buffer +#define ZT_NODE_PRNG_BUF_SIZE 64 + namespace ZeroTier { /** @@ -195,7 +198,7 @@ public: #endif bool shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); - inline bool getPathHint(const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } + inline bool externalPathLookup(const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); void postCircuitTestReport(const ZT_CircuitTestReport *report); @@ -284,7 +287,7 @@ private: unsigned int _prngStreamPtr; Salsa20 _prng; - uint64_t _prngStream[16]; // repeatedly encrypted with _prng to yield a high-quality non-crypto PRNG stream + uint64_t _prngStream[ZT_NODE_PRNG_BUF_SIZE]; // repeatedly encrypted with _prng to yield a high-quality non-crypto PRNG stream uint64_t _now; uint64_t _lastPingCheck; diff --git a/node/Peer.cpp b/node/Peer.cpp index e0bd0eac..2ef139e1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -43,6 +43,7 @@ static uint32_t _natKeepaliveBuf = 0; Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : _lastReceive(0), _lastNontrivialReceive(0), + _lastTriedMemorizedPath(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), _lastCredentialRequestSent(0), @@ -373,6 +374,16 @@ void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &at } } +void Peer::tryMemorizedPath(uint64_t now) +{ + if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) { + _lastTriedMemorizedPath = now; + InetAddress mp; + if (RR->node->externalPathLookup(_id.address(),-1,mp)) + attemptToContactAt(InetAddress(),mp,now); + } +} + bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); diff --git a/node/Peer.hpp b/node/Peer.hpp index a7240cb4..78b345b9 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -164,6 +164,13 @@ public: */ void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now); + /** + * Try a memorized or statically defined path if any are known + * + * Under the hood this is done periodically based on ZT_TRY_MEMORIZED_PATH_INTERVAL. + */ + void tryMemorizedPath(uint64_t now); + /** * Send pings or keepalives depending on configured timeouts * @@ -435,6 +442,7 @@ private: uint8_t _remoteClusterOptimal6[16]; uint64_t _lastReceive; // direct or indirect uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. + uint64_t _lastTriedMemorizedPath; uint64_t _lastDirectPathPushSent; uint64_t _lastDirectPathPushReceive; uint64_t _lastCredentialRequestSent; diff --git a/node/Switch.cpp b/node/Switch.cpp index 881d7b92..7c94d438 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -710,12 +710,12 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) if (peer) { const uint64_t now = RR->node->now(); - // First get the best path, and if it's dead (and this is not a root) - // we attempt to re-activate that path but this packet will flow - // upstream. If the path comes back alive, it will be used in the future. - // For roots we don't do the alive check since roots are not required - // to send heartbeats "down" and because we have to at least try to - // go somewhere. + /* First get the best path, and if it's dead (and this is not a root) + * we attempt to re-activate that path but this packet will flow + * upstream. If the path comes back alive, it will be used in the future. + * For roots we don't do the alive check since roots are not required + * to send heartbeats "down" and because we have to at least try to + * go somewhere. */ SharedPtr viaPath(peer->getBestPath(now,false)); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) { @@ -724,7 +724,8 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) viaPath.zero(); } if (!viaPath) { - SharedPtr relay(RR->topology->getUpstreamPeer()); + peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known + const SharedPtr relay(RR->topology->getUpstreamPeer()); if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { if (!(viaPath = peer->getBestPath(now,true))) return false; diff --git a/service/OneService.cpp b/service/OneService.cpp index 7434ca67..a2024e52 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1163,6 +1163,7 @@ public: // Internal implementation methods ----------------------------------------- + // Must be called after _localConfig is read or modified void applyLocalConfig() { Mutex::Lock _l(_localConfig_m); @@ -1872,6 +1873,12 @@ public: } } + /* Note: I do not think we need to scan for overlap with managed routes + * because of the "route forking" and interface binding that we do. This + * ensures (we hope) that ZeroTier traffic will still take the physical + * path even if its managed routes override this for other traffic. Will + * revisit if we see recursion problems. */ + // Check blacklists const Hashtable< uint64_t,std::vector > *blh = (const Hashtable< uint64_t,std::vector > *)0; const std::vector *gbl = (const std::vector *)0; @@ -1897,12 +1904,6 @@ public: } } - /* Note: I do not think we need to scan for overlap with managed routes - * because of the "route forking" and interface binding that we do. This - * ensures (we hope) that ZeroTier traffic will still take the physical - * path even if its managed routes override this for other traffic. Will - * revisit if we see problems with this. */ - return 1; } -- cgit v1.2.3 From fa2bb91ae5515ef18ed662896b83d654d659e9f1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 30 Nov 2016 10:48:09 -0800 Subject: Kill some old debug code. --- node/Topology.cpp | 5 +---- node/Utils.cpp | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index 5aafc439..517934fb 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -128,10 +128,7 @@ SharedPtr Topology::getPeer(const Address &zta) return ap; } } - } catch ( ... ) { - fprintf(stderr,"EXCEPTION in getPeer() part 2\n"); - abort(); - } // invalid identity on disk? + } catch ( ... ) {} // invalid identity on disk? return SharedPtr(); } diff --git a/node/Utils.cpp b/node/Utils.cpp index 215e3b3e..06b726cc 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -248,7 +248,6 @@ unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...) if ((n >= (int)len)||(n < 0)) { if (len) buf[len - 1] = (char)0; - abort(); throw std::length_error("buf[] overflow in Utils::snprintf"); } -- cgit v1.2.3 From 244f37179cb20b1ebec420da5b315ecf8ac0db40 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 5 Dec 2016 16:09:42 -0800 Subject: Minor security: lock roots to only be reachable via World IPs. --- node/Node.cpp | 3 +++ node/Topology.cpp | 17 +++++++++++++++++ node/Topology.hpp | 16 ++++++++++++++++ 3 files changed, 36 insertions(+) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 11f76365..ed60817f 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -633,6 +633,9 @@ bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddre if (!Path::isAddressValidForPath(remoteAddress)) return false; + if (RR->topology->isProhibitedEndpoint(ztaddr,remoteAddress)) + return false; + { Mutex::Lock _l(_networks_m); for(std::vector< std::pair< uint64_t, SharedPtr > >::const_iterator i=_networks.begin();i!=_networks.end();++i) { diff --git a/node/Topology.cpp b/node/Topology.cpp index 517934fb..bf51b585 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -264,6 +264,23 @@ void Topology::setUpstream(const Address &a,bool upstream) RR->sw->requestWhois(a); } +bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const +{ + Mutex::Lock _l(_lock); + + if (std::find(_rootAddresses.begin(),_rootAddresses.end(),ztaddr) != _rootAddresses.end()) { + for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { + for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { + if (ipaddr.ipsEqual(*e)) + return false; + } + } + return true; + } + + return false; +} + bool Topology::worldUpdateIfValid(const World &newWorld) { Mutex::Lock _l(_lock); diff --git a/node/Topology.hpp b/node/Topology.hpp index 8e1d28cb..90ad7083 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -163,6 +163,22 @@ public: */ void setUpstream(const Address &a,bool upstream); + /** + * Check for prohibited endpoints + * + * Right now this returns true if the designated ZT address is a root and if + * the IP (IP only, not port) does not equal any of the IPs defined in the + * current World. This is an extra little security feature in case root keys + * get appropriated or something. + * + * Otherwise it returns false. + * + * @param ztaddr ZeroTier address + * @param ipaddr IP address + * @return True if this ZT/IP pair should not be allowed to be used + */ + bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const; + /** * @return Vector of active upstream addresses (including roots) */ -- cgit v1.2.3 From 2eaff6d4843f420ff63c1b7b42e86028fa162f80 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 16:36:38 -0800 Subject: Fix to characteristcs in rules engine. --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 1f8e7ebf..e8b103ba 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -498,7 +498,7 @@ static _doZtFilterResult _doZtFilter( } } } - thisRuleMatches = (uint8_t)((cf | rules[rn].v.characteristics) != 0); + thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics) != 0); FILTER_TRACE("%u %s %c (%.16llx | %.16llx)!=0 -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics,(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: -- cgit v1.2.3 From fe530548bbc8d4d0e274f814718fad579a012812 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 16:57:45 -0800 Subject: Fix MATCH_RANDOM in controller. --- controller/EmbeddedNetworkController.cpp | 1 + node/Network.cpp | 1 + 2 files changed, 2 insertions(+) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index df20d4ce..01a7152c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -428,6 +428,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_RANDOM") { rule.t |= ZT_NETWORK_RULE_MATCH_RANDOM; rule.v.randomProbability = (uint32_t)(_jI(r["probability"],0ULL) & 0xffffffffULL); + return true; } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; rule.v.tag.id = (uint32_t)(_jI(r["id"],0ULL) & 0xffffffffULL); diff --git a/node/Network.cpp b/node/Network.cpp index e8b103ba..2488bea2 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -111,6 +111,7 @@ static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,b ); if (msg) printf(" + (%s)" ZT_EOL_S,msg); + fflush(stdout); } #else #define FILTER_TRACE(f,...) {} -- cgit v1.2.3 From 6b12d86209e9e2097d33c8ffee888294c6c4fba4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 18:06:35 -0800 Subject: Add a workaround for an edge case in TEE/REDIRECT if we are the inbound destination and teeing is only being done on the outbound side. --- node/Network.cpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 2488bea2..45ffc993 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -178,6 +178,9 @@ static _doZtFilterResult _doZtFilter( std::vector dlog; #endif // ZT_RULES_ENGINE_DEBUGGING + // Set to true if we are a TEE/REDIRECT/WATCH target + bool superAccept = false; + // The default match state for each set of entries starts as 'true' since an // ACTION with no MATCH entries preceding it is always taken. uint8_t thisSetMatches = 1; @@ -199,7 +202,7 @@ static _doZtFilterResult _doZtFilter( #ifdef ZT_RULES_ENGINE_DEBUGGING _dumpFilterTrace("ACTION_ACCEPT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); #endif // ZT_RULES_ENGINE_DEBUGGING - return DOZTFILTER_ACCEPT; // match, accept packet + return (superAccept ? DOZTFILTER_SUPER_ACCEPT : DOZTFILTER_ACCEPT); // match, accept packet // These are initially handled together since preliminary logic is common case ZT_NETWORK_RULE_ACTION_TEE: @@ -264,6 +267,22 @@ static _doZtFilterResult _doZtFilter( continue; } } else { + // If this is an incoming packet and we are a TEE or REDIRECT target, we should + // super-accept if we accept at all. This will cause us to accept redirected or + // tee'd packets in spite of MAC and ZT addressing checks. + if (inbound) { + switch(rt) { + case ZT_NETWORK_RULE_ACTION_TEE: + case ZT_NETWORK_RULE_ACTION_WATCH: + case ZT_NETWORK_RULE_ACTION_REDIRECT: + if (RR->identity.address() == rules[rn].v.fwd.address) + superAccept = true; + break; + default: + break; + } + } + #ifdef ZT_RULES_ENGINE_DEBUGGING _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); @@ -541,12 +560,12 @@ static _doZtFilterResult _doZtFilter( thisRuleMatches = 0; } } else { - if (inbound) { + if ((inbound)&&(!superAccept)) { thisRuleMatches = 0; FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { thisRuleMatches = 1; - FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side and TEE/REDIRECT targets are not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } } else { -- cgit v1.2.3 From c8554504f3a1203470a46749253564d3fe697ee3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 22 Dec 2016 18:37:46 -0800 Subject: . --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 45ffc993..8b0f2055 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -240,7 +240,7 @@ static _doZtFilterResult _doZtFilter( return DOZTFILTER_REDIRECT; } else { #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_TEE",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING cc = fwdAddr; -- cgit v1.2.3 From d5528e4e9a35d7f1c88a373b99c7b31a03eccd5a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 9 Jan 2017 15:55:07 -0800 Subject: Wire up VERB_USER_MESSAGE in core. --- include/ZeroTierOne.h | 52 ++++++++++++++++++++++++++++++++++++++++++++++++- node/IncomingPacket.cpp | 21 ++++++++++++++++++-- node/IncomingPacket.hpp | 1 + node/Node.cpp | 23 ++++++++++++++++++++++ node/Node.hpp | 1 + node/Packet.hpp | 4 ++++ 6 files changed, 99 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 21544b96..8b1ee0ac 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -393,7 +393,17 @@ enum ZT_Event * * Meta-data: C string, TRACE message */ - ZT_EVENT_TRACE = 5 + ZT_EVENT_TRACE = 5, + + /** + * VERB_USER_MESSAGE received + * + * These are generated when a VERB_USER_MESSAGE packet is received via + * ZeroTier VL1. + * + * Meta-data: ZT_UserMessage structure + */ + ZT_EVENT_USER_MESSAGE = 6 }; /** @@ -406,6 +416,32 @@ enum ZT_RelayPolicy ZT_RELAY_POLICY_ALWAYS = 2 }; +/** + * User message used with ZT_EVENT_USER_MESSAGE + */ +typedef struct +{ + /** + * ZeroTier address of sender (least significant 40 bits) + */ + uint64_t origin; + + /** + * User message type ID + */ + uint64_t typeId; + + /** + * User message data (not including type ID) + */ + const void *data; + + /** + * Length of data in bytes + */ + unsigned int length; +} ZT_UserMessage; + /** * Current node status */ @@ -1853,6 +1889,20 @@ int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage */ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node); +/** + * Send a VERB_USER_MESSAGE to another ZeroTier node + * + * There is no delivery guarantee here. Failure can occur if the message is + * too large or if dest is not a valid ZeroTier address. + * + * @param dest Destination ZeroTier address + * @param typeId VERB_USER_MESSAGE type ID + * @param data Payload data to attach to user message + * @param len Length of data in bytes + * @return Boolean: non-zero on success, zero on failure + */ +int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); + /** * Set peer role * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 7b828f8b..562aee91 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -106,8 +106,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); - case Packet::VERB_USER_MESSAGE: - return true; + case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,peer); } } else { RR->sw->requestWhois(sourceAddress); @@ -1345,6 +1344,24 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S return true; } +bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPtr &peer) +{ + try { + if (size() >= (ZT_PACKET_IDX_PAYLOAD + 8)) { + ZT_UserMessage um; + um.origin = peer->address().toInt(); + um.typeId = at(ZT_PACKET_IDX_PAYLOAD); + um.data = reinterpret_cast(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD + 8); + um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); + RR->node->postEvent(ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); + } + peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + } catch ( ... ) { + TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + } + return true; +} + void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid) { const uint64_t now = RR->node->now(); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 80244ea4..febff28a 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -131,6 +131,7 @@ private: bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer); bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); + bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPtr &peer); void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid); diff --git a/node/Node.cpp b/node/Node.cpp index ed60817f..32d41305 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -473,6 +473,20 @@ void Node::clearLocalInterfaceAddresses() _directPaths.clear(); } +int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len) +{ + try { + Packet outp(Address(dest),RR->identity.address(),Packet::VERB_USER_MESSAGE); + outp.append(typeId); + outp.append(data,len); + outp.compress(); + RR->sw->send(outp,true); + return 1; + } catch ( ... ) { + return 0; + } +} + void Node::setRole(uint64_t ztAddress,ZT_PeerRole role) { RR->topology->setUpstream(Address(ztAddress),(role == ZT_PEER_ROLE_UPSTREAM)); @@ -992,6 +1006,15 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node) } catch ( ... ) {} } +int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) +{ + try { + return reinterpret_cast(node)->sendUserMessage(dest,typeId,data,len); + } catch ( ... ) { + return 0; + } +} + void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role) { try { diff --git a/node/Node.hpp b/node/Node.hpp index eb46527d..64c9fcb4 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -98,6 +98,7 @@ public: void freeQueryResult(void *qr); int addLocalInterfaceAddress(const struct sockaddr_storage *addr); void clearLocalInterfaceAddresses(); + int sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setRole(uint64_t ztAddress,ZT_PeerRole role); void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); diff --git a/node/Packet.hpp b/node/Packet.hpp index 8ff817aa..5ecbecba 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1048,6 +1048,10 @@ public: * This can be used to send arbitrary messages over VL1. It generates no * OK or ERROR and has no special semantics outside of whatever the user * (via the ZeroTier core API) chooses to give it. + * + * Message type IDs less than or equal to 65535 are reserved for use by + * ZeroTier, Inc. itself. We recommend making up random ones for your own + * implementations. */ VERB_USER_MESSAGE = 0x14 }; -- cgit v1.2.3 From d7e7ad4f88ed8f5967568a9d5cec0716d1ae6265 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Jan 2017 17:46:52 -0800 Subject: Can't send a user message to self. --- node/Node.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 32d41305..0d0750ca 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -476,15 +476,16 @@ void Node::clearLocalInterfaceAddresses() int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len) { try { - Packet outp(Address(dest),RR->identity.address(),Packet::VERB_USER_MESSAGE); - outp.append(typeId); - outp.append(data,len); - outp.compress(); - RR->sw->send(outp,true); - return 1; - } catch ( ... ) { - return 0; - } + if (RR->identity.address().toInt() != dest) { + Packet outp(Address(dest),RR->identity.address(),Packet::VERB_USER_MESSAGE); + outp.append(typeId); + outp.append(data,len); + outp.compress(); + RR->sw->send(outp,true); + return 1; + } + } catch ( ... ) {} + return 0; } void Node::setRole(uint64_t ztAddress,ZT_PeerRole role) -- cgit v1.2.3 From 1346e31a8ee962c40b1f18cb8ff1d5fe866744e3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Jan 2017 14:22:36 -0800 Subject: Windows build fixes, Software update fix, warning removal. --- node/Multicaster.hpp | 2 +- one.cpp | 4 ++-- service/SoftwareUpdater.cpp | 8 ++++---- windows/ZeroTierOne/ZeroTierOne.vcxproj | 8 ++++---- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 9 ++++++--- 5 files changed, 17 insertions(+), 14 deletions(-) (limited to 'node') diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 5c94cd3a..32dec9cf 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -220,7 +220,7 @@ private: { _GatherAuthKey() : member(0),networkId(0) {} _GatherAuthKey(const uint64_t nwid,const Address &a) : member(a.toInt()),networkId(nwid) {} - inline unsigned long hashCode() const { return (member ^ networkId); } + inline unsigned long hashCode() const { return (unsigned long)(member ^ networkId); } inline bool operator==(const _GatherAuthKey &k) const { return ((member == k.member)&&(networkId == k.networkId)); } uint64_t member; uint64_t networkId; diff --git a/one.cpp b/one.cpp index 1cda3fb1..3ebaa8fa 100644 --- a/one.cpp +++ b/one.cpp @@ -495,7 +495,7 @@ static int cli(int argc,char **argv) (std::string("/network/") + arg1).c_str(), requestHeaders, jsons, - strlen(jsons), + (unsigned long)strlen(jsons), responseHeaders, responseBody); if (scode == 200) { @@ -572,7 +572,7 @@ static int idtool(int argc,char **argv) int vanityBits = 0; if (argc >= 5) { vanity = Utils::hexStrToU64(argv[4]) & 0xffffffffffULL; - vanityBits = 4 * strlen(argv[4]); + vanityBits = 4 * (int)strlen(argv[4]); if (vanityBits > 40) vanityBits = 40; } diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 299c0d97..f6572cfc 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -21,6 +21,9 @@ #include #include +#include "../node/Constants.hpp" +#include "../version.h" + #ifdef __WINDOWS__ #include #include @@ -37,9 +40,6 @@ #include "SoftwareUpdater.hpp" -#include "../version.h" - -#include "../node/Constants.hpp" #include "../node/Utils.hpp" #include "../node/SHA512.hpp" #include "../node/Buffer.hpp" @@ -369,7 +369,7 @@ void SoftwareUpdater::apply() PROCESS_INFORMATION pi; memset(&si,0,sizeof(si)); memset(&pi,0,sizeof(pi)); - CreateProcessA(NULL,updatePath.c_str(),NULL,NULL,FALSE,CREATE_NO_WINDOW|CREATE_NEW_PROCESS_GROUP,NULL,NULL,&si,&pi); + CreateProcessA(NULL,const_cast(updatePath.c_str()),NULL,NULL,FALSE,CREATE_NO_WINDOW|CREATE_NEW_PROCESS_GROUP,NULL,NULL,&si,&pi); #else char *argv[256]; unsigned long ac = 0; diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index fc259ebf..2a6545eb 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -255,7 +255,7 @@ true - NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions) 4996 @@ -271,7 +271,7 @@ true - NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_RULES_ENGINE_DEBUGGING;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_RULES_ENGINE_DEBUGGING;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions) false 4996 @@ -291,7 +291,7 @@ true - STATICLIB;ZT_OFFICIAL_RELEASE;ZT_AUTO_UPDATE;ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;%(PreprocessorDefinitions) + STATICLIB;ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;ZT_SOFTWARE_UPDATE_DEFAULT="apply";%(PreprocessorDefinitions) MultiThreaded NoExtensions true @@ -317,7 +317,7 @@ true - STATICLIB;ZT_OFFICIAL_RELEASE;ZT_AUTO_UPDATE;ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;%(PreprocessorDefinitions) + STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="apply";ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;%(PreprocessorDefinitions) MultiThreaded NotSet true diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index 1fa39abd..8a426d4b 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -264,6 +264,9 @@ Source Files\controller + + Source Files\service + @@ -293,9 +296,6 @@ Header Files\service - - Header Files\service - Header Files\service @@ -554,6 +554,9 @@ Header Files\osdep + + Header Files\service + -- cgit v1.2.3 From 0fb3d1d58239837ae058c43d7269ce7fc33f5a7c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 18 Jan 2017 09:16:23 -0800 Subject: Add a build version for software update use so we can do very minor updates within a version. --- make-mac.mk | 3 ++- node/Utils.hpp | 11 ++++++++--- service/SoftwareUpdater.cpp | 16 ++++++++++++---- service/SoftwareUpdater.hpp | 1 + version.h | 9 +++++++++ 5 files changed, 32 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/make-mac.mk b/make-mac.mk index 51fdd7ec..298d1a2e 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -14,6 +14,7 @@ ZT_BUILD_ARCHITECTURE=2 ZT_VERSION_MAJOR=$(shell cat version.h | grep -F VERSION_MAJOR | cut -d ' ' -f 3) ZT_VERSION_MINOR=$(shell cat version.h | grep -F VERSION_MINOR | cut -d ' ' -f 3) ZT_VERSION_REV=$(shell cat version.h | grep -F VERSION_REVISION | cut -d ' ' -f 3) +ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3) DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) @@ -84,7 +85,7 @@ mac-dist-pkg: FORCE $(PRODUCTSIGN) --sign $(CODESIGN_INSTALLER_CERT) "ZeroTier One.pkg" "ZeroTier One Signed.pkg" if [ -f "ZeroTier One Signed.pkg" ]; then mv -f "ZeroTier One Signed.pkg" "ZeroTier One.pkg"; fi rm -f zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* - cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV) + cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV)_$(ZT_VERSION_BUILD) # For ZeroTier, Inc. to build official signed packages official: FORCE diff --git a/node/Utils.hpp b/node/Utils.hpp index 48c43da3..48cf799e 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -335,8 +335,7 @@ public: * * @return -1, 0, or 1 based on whether first tuple is less than, equal to, or greater than second */ - static inline int compareVersion(unsigned int maj1,unsigned int min1,unsigned int rev1,unsigned int maj2,unsigned int min2,unsigned int rev2) - throw() + static inline int compareVersion(unsigned int maj1,unsigned int min1,unsigned int rev1,unsigned int b1,unsigned int maj2,unsigned int min2,unsigned int rev2,unsigned int b2) { if (maj1 > maj2) return 1; @@ -352,7 +351,13 @@ public: return 1; else if (rev1 < rev2) return -1; - else return 0; + else { + if (b1 > b2) + return 1; + else if (b1 < b2) + return -1; + else return 0; + } } } } diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index f6572cfc..c650ce42 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -74,7 +74,9 @@ SoftwareUpdater::SoftwareUpdater(Node &node,const std::string &homePath) : const unsigned int rvMaj = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); const unsigned int rvMin = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int rvRev = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); - if ((Utils::compareVersion(rvMaj,rvMin,rvRev,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION) > 0)&&(OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S ZT_SOFTWARE_UPDATE_BIN_FILENAME).c_str(),buf))) { + const unsigned int rvBld = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); + if ((Utils::compareVersion(rvMaj,rvMin,rvRev,rvBld,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD) > 0)&& + (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S ZT_SOFTWARE_UPDATE_BIN_FILENAME).c_str(),buf))) { if ((uint64_t)buf.length() == OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE],0)) { _latestMeta = meta; _latestValid = true; @@ -149,6 +151,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void const unsigned int rvMaj = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); const unsigned int rvMin = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int rvRev = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); + const unsigned int rvBld = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); const unsigned int rvPlatform = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_PLATFORM],0); const unsigned int rvArch = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE],0); const unsigned int rvVendor = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VENDOR],0); @@ -161,6 +164,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void unsigned int bestVMaj = rvMaj; unsigned int bestVMin = rvMin; unsigned int bestVRev = rvRev; + unsigned int bestVBld = rvBld; for(std::map< Array,_D >::const_iterator d(_dist.begin());d!=_dist.end();++d) { if ((OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_PLATFORM],0) == rvPlatform)&& (OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE],0) == rvArch)&& @@ -170,11 +174,13 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void const unsigned int dvMaj = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); const unsigned int dvMin = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int dvRev = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); - if (Utils::compareVersion(dvMaj,dvMin,dvRev,bestVMaj,bestVMin,bestVRev) > 0) { + const unsigned int dvBld = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); + if (Utils::compareVersion(dvMaj,dvMin,dvRev,dvBld,bestVMaj,bestVMin,bestVRev,bestVBld) > 0) { latest = &(d->second.meta); bestVMaj = dvMaj; bestVMin = dvMin; bestVRev = dvRev; + bestVBld = dvBld; } } } @@ -184,7 +190,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void lj.append(OSUtils::jsonDump(*latest)); _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); if (_distLog) { - fprintf(_distLog,"%.10llx GET_LATEST %u.%u.%u platform %u arch %u vendor %u channel %s -> LATEST %u.%u.%u" ZT_EOL_S,(unsigned long long)origin,rvMaj,rvMin,rvRev,rvPlatform,rvArch,rvVendor,rvChannel.c_str(),bestVMaj,bestVMin,bestVRev); + fprintf(_distLog,"%.10llx GET_LATEST %u.%u.%u_%u platform %u arch %u vendor %u channel %s -> LATEST %u.%u.%u_%u" ZT_EOL_S,(unsigned long long)origin,rvMaj,rvMin,rvRev,rvBld,rvPlatform,rvArch,rvVendor,rvChannel.c_str(),bestVMaj,bestVMin,bestVRev,bestVBld); fflush(_distLog); } } @@ -193,7 +199,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void } else { // VERB_LATEST if ((origin == ZT_SOFTWARE_UPDATE_SERVICE)&& - (Utils::compareVersion(rvMaj,rvMin,rvRev,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION) > 0)&& + (Utils::compareVersion(rvMaj,rvMin,rvRev,rvBld,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD) > 0)&& (OSUtils::jsonString(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY],"") == ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY)) { const unsigned long len = (unsigned long)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE],0); const std::string hash = OSUtils::jsonBinFromHex(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); @@ -283,6 +289,7 @@ bool SoftwareUpdater::check(const uint64_t now) "%c{\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "\":%d," + "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_EXPECT_SIGNED_BY "\":\"%s\"," "\"" ZT_SOFTWARE_UPDATE_JSON_PLATFORM "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE "\":%d," @@ -292,6 +299,7 @@ bool SoftwareUpdater::check(const uint64_t now) ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, ZEROTIER_ONE_VERSION_REVISION, + ZEROTIER_ONE_VERSION_BUILD, ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY, ZT_BUILD_PLATFORM, ZT_BUILD_ARCHITECTURE, diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index 0a7b6789..e6e6b0c7 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -85,6 +85,7 @@ #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "versionMajor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "versionMinor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "versionRev" +#define ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD "versionBuild" #define ZT_SOFTWARE_UPDATE_JSON_PLATFORM "platform" #define ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE "arch" #define ZT_SOFTWARE_UPDATE_JSON_VENDOR "vendor" diff --git a/version.h b/version.h index 118ac761..9f616664 100644 --- a/version.h +++ b/version.h @@ -34,4 +34,13 @@ */ #define ZEROTIER_ONE_VERSION_REVISION 17 +/** + * Build version + * + * This starts at 0 for each major.minor.rev tuple and can be incremented + * to force a minor update without an actual version number change. It's + * not part of the actual release version number. + */ +#define ZEROTIER_ONE_VERSION_BUILD 0 + #endif -- cgit v1.2.3 From 7612bf3302eef8a49658eaeca09504be588f6398 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 19 Jan 2017 14:54:39 -0800 Subject: Fix LZ4 warning. --- ext/http-parser/http_parser.c | 11 ++++--- ext/http-parser/http_parser.h | 72 ++++++++++++++++++++++++++++++++++++++++++- make-linux.mk | 31 ++++++++++--------- node/Packet.cpp | 2 +- node/Packet.hpp | 8 ++--- 5 files changed, 99 insertions(+), 25 deletions(-) (limited to 'node') diff --git a/ext/http-parser/http_parser.c b/ext/http-parser/http_parser.c index 3c896ffa..895bf0c7 100644 --- a/ext/http-parser/http_parser.c +++ b/ext/http-parser/http_parser.c @@ -1366,12 +1366,7 @@ reexecute: || c != CONTENT_LENGTH[parser->index]) { parser->header_state = h_general; } else if (parser->index == sizeof(CONTENT_LENGTH)-2) { - if (parser->flags & F_CONTENTLENGTH) { - SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); - goto error; - } parser->header_state = h_content_length; - parser->flags |= F_CONTENTLENGTH; } break; @@ -1474,6 +1469,12 @@ reexecute: goto error; } + if (parser->flags & F_CONTENTLENGTH) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } + + parser->flags |= F_CONTENTLENGTH; parser->content_length = ch - '0'; break; diff --git a/ext/http-parser/http_parser.h b/ext/http-parser/http_parser.h index 105ae510..45c72a07 100644 --- a/ext/http-parser/http_parser.h +++ b/ext/http-parser/http_parser.h @@ -27,7 +27,7 @@ extern "C" { /* Also update SONAME in the Makefile whenever you change these. */ #define HTTP_PARSER_VERSION_MAJOR 2 #define HTTP_PARSER_VERSION_MINOR 7 -#define HTTP_PARSER_VERSION_PATCH 0 +#define HTTP_PARSER_VERSION_PATCH 1 #include #if defined(_WIN32) && !defined(__MINGW32__) && \ @@ -90,6 +90,76 @@ typedef int (*http_data_cb) (http_parser*, const char *at, size_t length); typedef int (*http_cb) (http_parser*); +/* Status Codes */ +#define HTTP_STATUS_MAP(XX) \ + XX(100, CONTINUE, Continue) \ + XX(101, SWITCHING_PROTOCOLS, Switching Protocols) \ + XX(102, PROCESSING, Processing) \ + XX(200, OK, OK) \ + XX(201, CREATED, Created) \ + XX(202, ACCEPTED, Accepted) \ + XX(203, NON_AUTHORITATIVE_INFORMATION, Non-Authoritative Information) \ + XX(204, NO_CONTENT, No Content) \ + XX(205, RESET_CONTENT, Reset Content) \ + XX(206, PARTIAL_CONTENT, Partial Content) \ + XX(207, MULTI_STATUS, Multi-Status) \ + XX(208, ALREADY_REPORTED, Already Reported) \ + XX(226, IM_USED, IM Used) \ + XX(300, MULTIPLE_CHOICES, Multiple Choices) \ + XX(301, MOVED_PERMANENTLY, Moved Permanently) \ + XX(302, FOUND, Found) \ + XX(303, SEE_OTHER, See Other) \ + XX(304, NOT_MODIFIED, Not Modified) \ + XX(305, USE_PROXY, Use Proxy) \ + XX(307, TEMPORARY_REDIRECT, Temporary Redirect) \ + XX(308, PERMANENT_REDIRECT, Permanent Redirect) \ + XX(400, BAD_REQUEST, Bad Request) \ + XX(401, UNAUTHORIZED, Unauthorized) \ + XX(402, PAYMENT_REQUIRED, Payment Required) \ + XX(403, FORBIDDEN, Forbidden) \ + XX(404, NOT_FOUND, Not Found) \ + XX(405, METHOD_NOT_ALLOWED, Method Not Allowed) \ + XX(406, NOT_ACCEPTABLE, Not Acceptable) \ + XX(407, PROXY_AUTHENTICATION_REQUIRED, Proxy Authentication Required) \ + XX(408, REQUEST_TIMEOUT, Request Timeout) \ + XX(409, CONFLICT, Conflict) \ + XX(410, GONE, Gone) \ + XX(411, LENGTH_REQUIRED, Length Required) \ + XX(412, PRECONDITION_FAILED, Precondition Failed) \ + XX(413, PAYLOAD_TOO_LARGE, Payload Too Large) \ + XX(414, URI_TOO_LONG, URI Too Long) \ + XX(415, UNSUPPORTED_MEDIA_TYPE, Unsupported Media Type) \ + XX(416, RANGE_NOT_SATISFIABLE, Range Not Satisfiable) \ + XX(417, EXPECTATION_FAILED, Expectation Failed) \ + XX(421, MISDIRECTED_REQUEST, Misdirected Request) \ + XX(422, UNPROCESSABLE_ENTITY, Unprocessable Entity) \ + XX(423, LOCKED, Locked) \ + XX(424, FAILED_DEPENDENCY, Failed Dependency) \ + XX(426, UPGRADE_REQUIRED, Upgrade Required) \ + XX(428, PRECONDITION_REQUIRED, Precondition Required) \ + XX(429, TOO_MANY_REQUESTS, Too Many Requests) \ + XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, Request Header Fields Too Large) \ + XX(451, UNAVAILABLE_FOR_LEGAL_REASONS, Unavailable For Legal Reasons) \ + XX(500, INTERNAL_SERVER_ERROR, Internal Server Error) \ + XX(501, NOT_IMPLEMENTED, Not Implemented) \ + XX(502, BAD_GATEWAY, Bad Gateway) \ + XX(503, SERVICE_UNAVAILABLE, Service Unavailable) \ + XX(504, GATEWAY_TIMEOUT, Gateway Timeout) \ + XX(505, HTTP_VERSION_NOT_SUPPORTED, HTTP Version Not Supported) \ + XX(506, VARIANT_ALSO_NEGOTIATES, Variant Also Negotiates) \ + XX(507, INSUFFICIENT_STORAGE, Insufficient Storage) \ + XX(508, LOOP_DETECTED, Loop Detected) \ + XX(510, NOT_EXTENDED, Not Extended) \ + XX(511, NETWORK_AUTHENTICATION_REQUIRED, Network Authentication Required) \ + +enum http_status + { +#define XX(num, name, string) HTTP_STATUS_##name = num, + HTTP_STATUS_MAP(XX) +#undef XX + }; + + /* Request Methods */ #define HTTP_METHOD_MAP(XX) \ XX(0, DELETE, DELETE) \ diff --git a/make-linux.mk b/make-linux.mk index cb91ab88..6f116bfc 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -14,20 +14,23 @@ DESTDIR?= include objects.mk -# On Linux we auto-detect the presence of some libraries and if present we -# link against the system version. This works with our package build images. -ifeq ($(wildcard /usr/include/lz4.h),) - OBJS+=ext/lz4/lz4.o -else - LDLIBS+=-llz4 - DEFS+=-DZT_USE_SYSTEM_LZ4 -endif -ifeq ($(wildcard /usr/include/http_parser.h),) - OBJS+=ext/http-parser/http_parser.o -else - LDLIBS+=-lhttp_parser - DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER -endif +# Used to auto-detect these and use them if dev headers were present, but stopped +# since it caused too many damn problems. The http-parser library in particular +# is basically broken between versions. Fark the Debian policies about including +# libraries. It's better if things work. +#ifeq ($(wildcard /usr/include/lz4.h),) +# OBJS+=ext/lz4/lz4.o +#else +# LDLIBS+=-llz4 +# DEFS+=-DZT_USE_SYSTEM_LZ4 +#endif +#ifeq ($(wildcard /usr/include/http_parser.h),) +# OBJS+=ext/http-parser/http_parser.o +#else +# LDLIBS+=-lhttp_parser +# DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER +#endif +OBJS+=ext/lz4/lz4.o ext/http-parser/http_parser.o # Auto-detect miniupnpc and nat-pmp as well and use system libs if present, # otherwise build into binary as done on Mac and Windows. diff --git a/node/Packet.cpp b/node/Packet.cpp index bdbc5fe4..6d1f49e8 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -125,7 +125,7 @@ bool Packet::compress() unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) { int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl); + int cl = LZ4_compress_default((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2); if ((cl > 0)&&(cl < pl)) { (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); diff --git a/node/Packet.hpp b/node/Packet.hpp index 5ecbecba..0ac5d9d3 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -34,11 +34,11 @@ #include "Utils.hpp" #include "Buffer.hpp" -#ifdef ZT_USE_SYSTEM_LZ4 -#include -#else +//#ifdef ZT_USE_SYSTEM_LZ4 +//#include +//#else #include "../ext/lz4/lz4.h" -#endif +//#endif /** * Protocol version -- incremented only for major changes -- cgit v1.2.3 From 0995c1dcaa02a61a65e447f56aec1353885530df Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 19 Jan 2017 15:16:04 -0800 Subject: Encapsulate LZ4 in Packet.cpp to eliminate dependency. --- ext/lz4/LICENSE | 24 - ext/lz4/README.md | 73 --- ext/lz4/lz4.c | 1463 ----------------------------------------- ext/lz4/lz4.h | 463 ------------- make-freebsd.mk | 6 +- make-linux.mk | 12 +- make-mac.mk | 6 +- node/Packet.cpp | 1891 ++++++++++++++++++++++++++++++++++++++++++++++++++++- node/Packet.hpp | 2 +- 9 files changed, 1900 insertions(+), 2040 deletions(-) delete mode 100644 ext/lz4/LICENSE delete mode 100644 ext/lz4/README.md delete mode 100644 ext/lz4/lz4.c delete mode 100644 ext/lz4/lz4.h (limited to 'node') diff --git a/ext/lz4/LICENSE b/ext/lz4/LICENSE deleted file mode 100644 index 74c2cdd7..00000000 --- a/ext/lz4/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ext/lz4/README.md b/ext/lz4/README.md deleted file mode 100644 index b40442c4..00000000 --- a/ext/lz4/README.md +++ /dev/null @@ -1,73 +0,0 @@ -LZ4 - Library Files -================================ - -The directory contains many files, but depending on project's objectives, -not all of them are necessary. - -#### Minimal LZ4 build - -The minimum required is **`lz4.c`** and **`lz4.h`**, -which will provide the fast compression and decompression algorithm. - - -#### The High Compression variant of LZ4 - -For more compression at the cost of compression speed, -the High Compression variant **lz4hc** is available. -It's necessary to add **`lz4hc.c`** and **`lz4hc.h`**. -The variant still depends on regular `lz4` source files. -In particular, the decompression is still provided by `lz4.c`. - - -#### Compatibility issues - -In order to produce files or streams compatible with `lz4` command line utility, -it's necessary to encode lz4-compressed blocks using the [official interoperable frame format]. -This format is generated and decoded automatically by the **lz4frame** library. -In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**, -which provides error detection. -(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. -This is what `liblz4` does, to avoid symbol duplication -in case a user program would link to several libraries containing xxhash symbols.) - - -#### Advanced API - -A more complex `lz4frame_static.h` is also provided. -It contains definitions which are not guaranteed to remain stable within future versions. -It must be used with static linking ***only***. - - -#### Using MinGW+MSYS to create DLL - -DLL can be created using MinGW+MSYS with the `make liblz4` command. -This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. -The import library is only required with Visual C++. -The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library -`dll\liblz4.dll` are required to compile a project using gcc/MinGW. -The dynamic library has to be added to linking options. -It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be linked with `dll\liblz4.dll`. For example: -``` - gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll -``` -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. - - -#### Miscellaneous - -Other files present in the directory are not source code. There are : - - - LICENSE : contains the BSD license text - - Makefile : script to compile or install lz4 library (static or dynamic) - - liblz4.pc.in : for pkg-config (make install) - - README.md : this file - -[official interoperable frame format]: ../doc/lz4_Frame_format.md - - -#### License - -All source material within __lib__ directory are BSD 2-Clause licensed. -See [LICENSE](LICENSE) for details. -The license is also repeated at the top of each source file. diff --git a/ext/lz4/lz4.c b/ext/lz4/lz4.c deleted file mode 100644 index 143c36e1..00000000 --- a/ext/lz4/lz4.c +++ /dev/null @@ -1,1463 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - - -/*-************************************ -* Tuning parameters -**************************************/ -/* - * HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). - */ -#ifndef HEAPMODE -# define HEAPMODE 0 -#endif - -/* - * ACCELERATION_DEFAULT : - * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 - */ -#define ACCELERATION_DEFAULT 1 - - -/*-************************************ -* CPU Feature Detection -**************************************/ -/* LZ4_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which generate assembly depending on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define LZ4_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/* - * LZ4_FORCE_SW_BITCOUNT - * Define this parameter if your target system or compiler does not support hardware bit count - */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/*-************************************ -* Dependency -**************************************/ -#include "lz4.h" -/* see also "memory routines" below */ - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#else -# if defined(__GNUC__) || defined(__clang__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define FORCE_INLINE static inline -# else -# define FORCE_INLINE static -# endif -#endif /* _MSC_VER */ - -#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/*-************************************ -* Basic Types -**************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef uintptr_t uptrval; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef size_t uptrval; /* generally true, except OpenVMS-64 */ -#endif - -#if defined(__x86_64__) - typedef U64 reg_t; /* 64-bits in x32 mode */ -#else - typedef size_t reg_t; /* 32-bits in x32 mode */ -#endif - -/*-************************************ -* Reading and writing into memory -**************************************/ -static unsigned LZ4_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - - -#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) -/* lie to the compiler about data alignment; use with caution */ - -static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } -static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } -static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } - -static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } - -#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; - -static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } - -static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } - -#else /* safe and portable access through memcpy() */ - -static U16 LZ4_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static U32 LZ4_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static reg_t LZ4_read_ARCH(const void* memPtr) -{ - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static void LZ4_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -static void LZ4_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* LZ4_FORCE_MEMORY_ACCESS */ - - -static U16 LZ4_readLE16(const void* memPtr) -{ - if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); - } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); - } -} - -static void LZ4_writeLE16(void* memPtr, U16 value) -{ - if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - } -} - -static void LZ4_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - -/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else /* Big Endian CPU */ { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -#define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - while (likely(pIn compression run slower on incompressible data */ - - -/*-************************************ -* Local Structures and types -**************************************/ -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; - -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - - -/*-************************************ -* Local Utils -**************************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - - -/*-****************************** -* Compression functions -********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) -{ - if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); -} - -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) -{ - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; - const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); -} - -FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) -{ - if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); - return LZ4_hash4(LZ4_read32(p), tableType); -} - -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) -{ - switch (tableType) - { - case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } -} - -FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); -} - -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ -} - -FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); -} - - -/** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) -{ - const BYTE* ip = (const BYTE*) source; - const BYTE* base; - const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; - - U32 forwardH; - - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; - } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizehashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } - - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } - - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun<internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } -} - - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ -#if (HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; -#endif - - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); - -#if (HEAPMODE) - FREEMEM(ctxPtr); -#endif - return result; -} - - -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/*-****************************** -* *_destSize() variant -********************************/ - -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } -} - - -int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) -{ -#if (HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; -#endif - - int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); - -#if (HEAPMODE) - FREEMEM(ctx); -#endif - return result; -} - - - -/*-****************************** -* Streaming functions -********************************/ - -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; -} - -void LZ4_resetStream (LZ4_stream_t* LZ4_stream) -{ - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); -} - -int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return (0); -} - - -#define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) -{ - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; - - while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); - p+=3; - } - - return dict->dictSize; -} - - -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) -{ - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } -} - - -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - /* external dictionary mode */ - { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - - -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); - - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - - return result; -} - - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -{ - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; - - return dictSize; -} - - - -/*-***************************** -* Decompression functions -*******************************/ -/*! LZ4_decompress_generic() : - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is important this generic function is really inlined, - * in order to remove useless branches during compilation optimization. - */ -FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - /* Local Variables */ - const BYTE* ip = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; - - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; - - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - - /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - - /* Main Loop : decode sequences */ - while (1) { - size_t length; - const BYTE* match; - size_t offset; - - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); - } - op=cpy; /* correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-source); /* Nb of input bytes read */ - - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-source))-1; -} - - -int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); -} - -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); -} - -int LZ4_decompress_fast(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); -} - - -/*===== streaming decompression functions =====*/ - -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - */ -LZ4_streamDecode_t* LZ4_createStreamDecode(void) -{ - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); - return lz4s; -} - -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return 0; -} - -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error - */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() -*/ -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); -} - -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - - -/*=************************************************* -* Obsolete Functions -***************************************************/ -/* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } - -/* -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe -*/ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} - -int LZ4_resetStreamState(void* state, char* inputBuffer) -{ - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); - return 0; -} - -void* LZ4_create (char* inputBuffer) -{ - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; -} - -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/ext/lz4/lz4.h b/ext/lz4/lz4.h deleted file mode 100644 index 0aae19c9..00000000 --- a/ext/lz4/lz4.h +++ /dev/null @@ -1,463 +0,0 @@ -/* - * LZ4 - Fast LZ compression algorithm - * Header File - * Copyright (C) 2011-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ -#ifndef LZ4_H_2983827168210 -#define LZ4_H_2983827168210 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* --- Dependency --- */ -#include /* size_t */ - - -/** - Introduction - - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, - scalable with multi-cores CPU. It features an extremely fast decoder, with speed in - multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. - - The LZ4 compression library provides in-memory compression and decompression functions. - Compression can be done in: - - a single step (described as Simple Functions) - - a single step, reusing a context (described in Advanced Functions) - - unbounded multiple steps (described as Streaming compression) - - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-compressed block also requires metadata (such as compressed size). - Each application is free to encode such metadata in whichever way it wants. - - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. -*/ - -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LZ4LIB_API -#endif - - -/*========== Version =========== */ -#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ - -#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) - -#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE -#define LZ4_QUOTE(str) #str -#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) -#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) - -LZ4LIB_API int LZ4_versionNumber (void); -LZ4LIB_API const char* LZ4_versionString (void); - - -/*-************************************ -* Tuning parameter -**************************************/ -/*! - * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect - * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache - */ -#define LZ4_MEMORY_USAGE 14 - - -/*-************************************ -* Simple Functions -**************************************/ -/*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); - -/*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - - -/*-************************************ -* Advanced Functions -**************************************/ -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) - -/*! -LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) - inputSize : max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) -*/ -LZ4LIB_API int LZ4_compressBound(int inputSize); - -/*! -LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. - The larger the acceleration value, the faster the algorithm, but also the lesser the compression. - It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. -*/ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ -LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails -*/ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); - - -/*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); - - -/*-********************************************* -* Streaming Compression Functions -***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ - -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ -LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); -LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); - -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. - */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); - -/*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= 64 KB) - */ -LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); - -/*! LZ4_compress_fast_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to still be present and unmodified ! - * 'dst' buffer must be already allocated. - * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - */ -LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); - - -/*-********************************************** -* Streaming Decompression Functions -* Bufferless synchronous API -************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ - -/* creation / destruction of streaming decompression tracking structure */ -LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); -LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - -/*! LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * @return : 1 if OK, 0 if error - */ -LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); - -/*! -LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() -*/ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); - - -/*! LZ4_decompress_*_usingDict() : - * These decoding functions work the same as - * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() - * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); - - -/*^********************************************** - * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ - uint32_t dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#else - -typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#endif - -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) -union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; - LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ - - -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; - LZ4_streamDecode_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_streamDecode_t */ - - -/*=************************************ -* Obsolete Functions -**************************************/ -/* Deprecation warnings */ -/* Should these warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ -#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS -# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ -#else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) -# endif -#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ - -/* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* Obsolete decompression functions */ -/* These function names are completely deprecated and must no longer be used. - They are only provided in lz4.c for compatibility with older programs. - - LZ4_uncompress is the same as LZ4_decompress_fast - - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe - These function prototypes are now disabled; uncomment them only if you really need them. - It is highly recommended to stop using these prototypes and migrate to maintained ones */ -/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ -/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ - -/* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); - -/* Obsolete streaming decoding functions */ -LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); - - -#if defined (__cplusplus) -} -#endif - -#endif /* LZ4_H_2983827168210 */ diff --git a/make-freebsd.mk b/make-freebsd.mk index 5b2a7766..23ad278b 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -5,7 +5,7 @@ DEFS= LIBS= include objects.mk -OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o +OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o # Build with ZT_ENABLE_CLUSTER=1 to build with cluster support ifeq ($(ZT_ENABLE_CLUSTER),1) @@ -20,7 +20,7 @@ ifeq ($(ZT_DEBUG),1) STRIP=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. -ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else CFLAGS?=-O3 -fstack-protector CFLAGS+=-Wall -fPIE -fvisibility=hidden -fstack-protector -pthread $(INCLUDES) -DNDEBUG $(DEFS) @@ -69,7 +69,7 @@ selftest: $(OBJS) selftest.o $(STRIP) zerotier-selftest clean: - rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* + rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* debug: FORCE make -j 4 ZT_DEBUG=1 diff --git a/make-linux.mk b/make-linux.mk index 6f116bfc..5121d604 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -18,19 +18,13 @@ include objects.mk # since it caused too many damn problems. The http-parser library in particular # is basically broken between versions. Fark the Debian policies about including # libraries. It's better if things work. -#ifeq ($(wildcard /usr/include/lz4.h),) -# OBJS+=ext/lz4/lz4.o -#else -# LDLIBS+=-llz4 -# DEFS+=-DZT_USE_SYSTEM_LZ4 -#endif #ifeq ($(wildcard /usr/include/http_parser.h),) # OBJS+=ext/http-parser/http_parser.o #else # LDLIBS+=-lhttp_parser # DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER #endif -OBJS+=ext/lz4/lz4.o ext/http-parser/http_parser.o +OBJS+=ext/http-parser/http_parser.o # Auto-detect miniupnpc and nat-pmp as well and use system libs if present, # otherwise build into binary as done on Mac and Windows. @@ -71,7 +65,7 @@ ifeq ($(ZT_DEBUG),1) STRIP?=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! -ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else CFLAGS?=-O3 -fstack-protector override CFLAGS+=-Wall -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) @@ -132,7 +126,7 @@ manpages: FORCE doc: manpages clean: FORCE - rm -rf *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules + rm -rf *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules distclean: clean diff --git a/make-mac.mk b/make-mac.mk index 298d1a2e..6c388a73 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -19,7 +19,7 @@ ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3 DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o +OBJS+=osdep/OSXEthernetTap.o ext/http-parser/http_parser.o # Official releases are signed with our Apple cert and apply software updates by default ifeq ($(ZT_OFFICIAL_RELEASE),1) @@ -48,7 +48,7 @@ ifeq ($(ZT_DEBUG),1) STRIP=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. -ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else CFLAGS?=-Ofast -fstack-protector-strong CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) @@ -95,7 +95,7 @@ official: FORCE make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: - rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier mkworld doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* + rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier mkworld doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* distclean: clean diff --git a/node/Packet.cpp b/node/Packet.cpp index 6d1f49e8..05fe8dd9 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -16,10 +16,1899 @@ * along with this program. If not, see . */ +#include +#include + #include "Packet.hpp" namespace ZeroTier { +/************************************************************************** */ +/************************************************************************** */ + +/* LZ4 is shipped encapsulated into Packet in an anonymous namespace. + * + * We're doing this as a deliberate workaround for various Linux distribution + * policies that forbid static linking of support libraries. + * + * The reason is that relying on distribution versions of LZ4 has been too + * big a source of bugs and compatibility issues. The LZ4 API is not stable + * enough across versions, and dependency hell ensues. So fark it. */ + +/* Needless to say the code in this anonymous namespace should be considered + * BSD 2-clause licensed. */ + +namespace { + +/* lz4.h ------------------------------------------------------------------ */ + +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (C) 2011-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/* --- Dependency --- */ +//#include /* size_t */ + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h provides block compression functions. It gives full buffer control to user. + Decompressing an lz4-compressed block also requires metadata (such as compressed size). + Each application is free to encode such metadata in whichever way it wants. + + An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), + take care of encoding standard metadata alongside LZ4-compressed blocks. + If your application requires interoperability, it's recommended to use it. + A library is provided to take care of it, see lz4frame.h. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API +#endif + + +/*========== Version =========== */ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) + +//LZ4LIB_API int LZ4_versionNumber (void); +//LZ4LIB_API const char* LZ4_versionString (void); + + +/*-************************************ +* Tuning parameter +**************************************/ +/*! + * LZ4_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ +#define LZ4_MEMORY_USAGE 14 + + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails */ +//LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + +/*! LZ4_decompress_safe() : + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. +*/ +LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! +LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! +LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. +*/ +LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); + + +/*! +LZ4_compress_fast_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +//LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); + + +/*! +LZ4_compress_destSize() : + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails +*/ +//LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); + + +/*! +LZ4_decompress_fast() : + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ4_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). +*/ +//LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); + +/*! +LZ4_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +//LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/*! LZ4_createStream() and LZ4_freeStream() : + * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. + * LZ4_freeStream() releases its memory. + */ +//LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +//LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure can be allocated once and re-used multiple times. + * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to load a static dictionary into LZ4_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= 64 KB) + */ +//LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_compress_fast_continue() : + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +//LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +//LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ + +/* creation / destruction of streaming decompression tracking structure */ +//LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +//LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * @return : 1 if OK, 0 if error + */ +//LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! +LZ4_decompress_*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() +*/ +//LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +//LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); + + +/*! LZ4_decompress_*_usingDict() : + * These decoding functions work the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + */ +//LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +//LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ +/*-************************************ + * Private definitions + ************************************** + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Using these definitions will expose code to API and/or ABI break in future versions of the library. + **************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include + +typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + uint32_t dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#else + +typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + unsigned int dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#endif + +/*! + * LZ4_stream_t : + * information structure to track an LZ4 stream. + * init this structure before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +union LZ4_stream_u { + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_stream_t */ + + +/*! + * LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode (or memset()) before first use + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +union LZ4_streamDecode_u { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + +/* lz4.c ------------------------------------------------------------------ */ + +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef HEAPMODE +# define HEAPMODE 0 +#endif + +/* + * ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define ACCELERATION_DEFAULT 1 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which generate assembly depending on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +/*-************************************ +* Dependency +**************************************/ +//#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define FORCE_INLINE static inline +# else +# define FORCE_INLINE static +# endif +#endif /* _MSC_VER */ + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +/*-************************************ +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + + +/*-************************************ +* Basic Types +**************************************/ +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +/*-************************************ +* Reading and writing into memory +**************************************/ +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; + +static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } + +#else /* safe and portable access through memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +static void LZ4_copy8(void* dst, const void* src) +{ + memcpy(dst,src,8); +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +#define STEPSIZE sizeof(reg_t) +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; +typedef enum { byPtr, byU32, byU16 } tableType_t; + +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + + +/*-************************************ +* Local Utils +**************************************/ +//int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +//const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +//int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + + +/*-****************************** +* Compression functions +********************************/ +static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + static const U64 prime5bytes = 889523592379ULL; + static const U64 prime8bytes = 11400714785074694791ULL; + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + else + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); +} + +FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); +} + +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + + +/** LZ4_compress_generic() : + inlined, to ensure branches are decided at compilation time */ +FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) +{ + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - cctx->dictSize; + const BYTE* const dictionary = cctx->dictionary; + const BYTE* const dictEnd = dictionary + cctx->dictSize; + const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source - cctx->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSizehashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + ptrdiff_t refDelta = 0; + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; + if (litLength >= RUN_MASK) { + int len = (int)litLength-RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchCode; + if (ip==limit) { + unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchCode; + } + + if ( outputLimited && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) + return 0; + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { size_t const lastRun = (size_t)(iend - anchor); + if ( (outputLimited) && /* Check output buffer overflow */ + ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) + return 0; + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun<internal_donotuse; + LZ4_resetStream((LZ4_stream_t*)state); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } else { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ +#if (HEAPMODE) + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctx; + void* const ctxPtr = &ctx; +#endif + + int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + +#if (HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + +#if 0 +int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); +} + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t ctx; + LZ4_resetStream(&ctx); + + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); +} +#endif + +/*-****************************** +* *_destSize() variant +********************************/ + +#if 0 +static int LZ4_compress_destSize_generic( + LZ4_stream_t_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtrhashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = 1 << LZ4_skipTrigger; + + do { + U32 h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + /* Encode Literal length */ + { unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< oMaxMatch) { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } else { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); + } +} + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (HEAPMODE) + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} +#endif + +/*-****************************** +* Streaming functions +********************************/ + +#if 0 +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; +} +#endif + +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); +} + +#if 0 +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return (0); +} +#endif + +#if 0 +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); + + if (dictSize < (int)HASH_UNIT) { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + LZ4_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } + + return dict->dictSize; +} + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +{ + if ((LZ4_dict->currentOffset > 0x80000000) || + ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } +} + +/* Hidden debug function, to force external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT(streamPtr, smallest); + + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + + return result; +} + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + +#endif + +/*-***************************** +* Decompression functions +*******************************/ +/*! LZ4_decompress_generic() : + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is important this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int LZ4_decompress_generic( + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; + const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + /* Main Loop : decode sequences */ + while (1) { + size_t length; + const BYTE* match; + size_t offset; + + /* get literal length */ + unsigned const token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) { + unsigned s; + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + memcpy(op+4, match, 4); + match -= dec64; + } else { LZ4_copy8(op, match); match+=8; } + op += 8; + + if (unlikely(cpy>oend-12)) { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op16) LZ4_wildCopy(op+8, match+8, cpy); + } + op=cpy; /* correction */ + } + + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-source); /* Nb of input bytes read */ + + /* Overflow error detected */ +_output_error: + return (int) (-(((const char*)ip)-source))-1; +} + + +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); +} + +#if 0 +int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); +} + +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); +} +#endif + +/*===== streaming decompression functions =====*/ + +#if 0 +/* + * If you prefer dynamic allocation methods, + * LZ4_createStreamDecode() + * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. + */ +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + return lz4s; +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return 0; +} + +/*! + * LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * Return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); +} + +/* debug function */ +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +#endif + +#if 0 +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } + +/* +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } + +static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) +{ + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; +} + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + return 0; +} + +void* LZ4_create (char* inputBuffer) +{ + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); + return lz4ds; +} + +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; + int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); +} + +/* Obsolete streaming decompression functions */ + +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} +#endif + +#endif /* LZ4_COMMONDEFS_ONLY */ + +} // anonymous namespace + +/************************************************************************** */ +/************************************************************************** */ + const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; #ifdef ZT_TRACE @@ -125,7 +2014,7 @@ bool Packet::compress() unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) { int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_default((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2); + int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); if ((cl > 0)&&(cl < pl)) { (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); diff --git a/node/Packet.hpp b/node/Packet.hpp index 0ac5d9d3..0be19f8a 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -37,7 +37,7 @@ //#ifdef ZT_USE_SYSTEM_LZ4 //#include //#else -#include "../ext/lz4/lz4.h" +//#include "../ext/lz4/lz4.h" //#endif /** -- cgit v1.2.3 From 9a475eeff90a181af0661b87b09af7337e39167a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 20 Jan 2017 12:00:18 -0800 Subject: Windows build fix, warning removal. --- node/Node.hpp | 6 ++ windows/ZeroTierOne/ZeroTierOne.vcxproj | 20 +----- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 81 ++----------------------- 3 files changed, 13 insertions(+), 94 deletions(-) (limited to 'node') diff --git a/node/Node.hpp b/node/Node.hpp index 64c9fcb4..d7b039b8 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -65,6 +65,12 @@ public: Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); virtual ~Node(); + // Get rid of alignment warnings on 32-bit Windows and possibly improve performance +#ifdef __WINDOWS__ + void * operator new(size_t i) { return _mm_malloc(i,16); } + void operator delete(void* p) { _mm_free(p); } +#endif + // Public API Functions ---------------------------------------------------- ZT_ResultCode processWirePacket( diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 2a6545eb..6a2ca520 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -25,7 +25,6 @@ - @@ -81,28 +80,11 @@ - - - - - - - - - - - - - - - - - + - diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index 8a426d4b..ff3d4821 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -40,18 +40,9 @@ {17ae9a01-d39f-4c6d-a800-8f2cd0804c96} - - {736aad7f-8d95-4602-88df-3bb970869c6f} - - - {3636527c-bc03-4852-bd3c-20ee25e56d82} - {7784af31-5b60-4300-b07e-44cf864c54db} - - {29164186-10fc-45f5-b253-6d03f0ddd4db} - {f8a1c208-15b8-4d85-a4cb-11d2b82f2d1e} @@ -67,15 +58,6 @@ {bf604491-14c4-4a74-81a6-6105d07c5c7c} - - {5939db69-ab17-47c6-97fb-185e2c678737} - - - {3666f510-b6da-47cb-8039-56441f2dac3e} - - - {1a47071e-e51b-4535-89ae-858946f03118} - {5423fb64-896b-432e-a19d-88d4467f89f9} @@ -91,6 +73,9 @@ {3cad34c8-c436-43ae-8323-57803637c832} + + {ff20532b-d9a2-440d-a7b4-b49e26a9b2f8} + @@ -165,9 +150,6 @@ Source Files\node - - Source Files\ext\lz4 - Source Files\ext\http-parser @@ -404,12 +386,6 @@ Header Files\node - - Header Files\ext\lz4 - - - Header Files\ext\json-parser - Header Files\ext\http-parser @@ -425,54 +401,6 @@ Header Files\osdep - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - - - Header Files\ext\bin\miniupnpc\include - Header Files\node @@ -557,6 +485,9 @@ Header Files\service + + Header Files\ext\json + -- cgit v1.2.3 From 8f2a42d1ad84e5dba590e7f593d8a46cc81389b3 Mon Sep 17 00:00:00 2001 From: Michał Zieliński Date: Sun, 22 Jan 2017 23:02:34 +0100 Subject: allow user to specify arbitrary allowed IP networks in allowManaged --- node/Buffer.hpp | 10 +++++----- service/ControlPlane.cpp | 11 ++++++++++- service/OneService.cpp | 41 ++++++++++++++++++++++++++++++++++++++--- service/OneService.hpp | 7 +++++++ 4 files changed, 60 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/node/Buffer.hpp b/node/Buffer.hpp index 0b171592..1a478894 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -61,11 +61,11 @@ public: // STL container idioms typedef unsigned char value_type; typedef unsigned char * pointer; - typedef const unsigned char * const_pointer; - typedef unsigned char & reference; - typedef const unsigned char & const_reference; - typedef unsigned char * iterator; - typedef const unsigned char * const_iterator; + typedef const char * const_pointer; + typedef char & reference; + typedef const char & const_reference; + typedef char * iterator; + typedef const char * const_iterator; typedef unsigned int size_type; typedef int difference_type; typedef std::reverse_iterator reverse_iterator; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 86158a91..27027d3b 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -121,6 +121,15 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_VirtualNetw case ZT_NETWORK_TYPE_PUBLIC: ntype = "PUBLIC"; break; } + std::string allowManaged = (localSettings.allowManaged) ? "true" : "false"; + if (localSettings.allowManagedWhitelist.size() != 0) { + allowManaged = ""; + for (InetAddress address : localSettings.allowManagedWhitelist) { + if (allowManaged.size() != 0) allowManaged += ','; + allowManaged += address.toIpString() + "/" + std::to_string(address.netmaskBits()); + } + } + Utils::snprintf(json,sizeof(json), "%s{\n" "%s\t\"id\": \"%.16llx\",\n" @@ -158,7 +167,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_VirtualNetw prefix,_jsonEnumerate(nc->assignedAddresses,nc->assignedAddressCount).c_str(), prefix,_jsonEnumerate(nc->routes,nc->routeCount).c_str(), prefix,_jsonEscape(portDeviceName).c_str(), - prefix,(localSettings.allowManaged) ? "true" : "false", + prefix,allowManaged.c_str(), prefix,(localSettings.allowGlobal) ? "true" : "false", prefix,(localSettings.allowDefault) ? "true" : "false", prefix); diff --git a/service/OneService.cpp b/service/OneService.cpp index 93f5b5f0..603234a2 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1039,6 +1039,18 @@ public: { if (!n.settings.allowManaged) return false; + + if (n.settings.allowManagedWhitelist.size() > 0) { + bool allowed = false; + for (InetAddress addr : n.settings.allowManagedWhitelist) { + if (addr.containsAddress(target) && addr.netmaskBits() <= target.netmaskBits()) { + allowed = true; + break; + } + } + if (!allowed) return false; + } + if (target.isDefaultRoute()) return n.settings.allowDefault; switch(target.ipScope()) { @@ -1423,9 +1435,32 @@ public: if (OSUtils::readFile(nlcpath,nlcbuf)) { Dictionary<4096> nc; nc.load(nlcbuf.c_str()); - n.settings.allowManaged = nc.getB("allowManaged",true); - n.settings.allowGlobal = nc.getB("allowGlobal",false); - n.settings.allowDefault = nc.getB("allowDefault",false); + Buffer<1024> allowManaged; + if (nc.get("allowManaged", allowManaged) && allowManaged.size() != 0) { + std::string addresses (allowManaged.begin(), allowManaged.size()); + if (allowManaged.size() <= 5) { // untidy parsing for backward compatibility + if (allowManaged[0] == '1' || allowManaged[0] == 't' || allowManaged[0] == 'T') { + n.settings.allowManaged = true; + } else { + n.settings.allowManaged = false; + } + } else { + // this should be a list of IP addresses + n.settings.allowManaged = true; + size_t pos = 0; + while (true) { + size_t nextPos = addresses.find(',', pos); + std::string address = addresses.substr(pos, (nextPos == std::string::npos ? addresses.size() : nextPos) - pos); + n.settings.allowManagedWhitelist.push_back(InetAddress(address)); + if (nextPos == std::string::npos) break; + pos = nextPos + 1; + } + } + } else { + n.settings.allowManaged = true; + } + n.settings.allowGlobal = nc.getB("allowGlobal", false); + n.settings.allowDefault = nc.getB("allowDefault", false); } } catch (std::exception &exc) { #ifdef __WINDOWS__ diff --git a/service/OneService.hpp b/service/OneService.hpp index 7aa3b361..88225da4 100644 --- a/service/OneService.hpp +++ b/service/OneService.hpp @@ -20,6 +20,7 @@ #define ZT_ONESERVICE_HPP #include +#include namespace ZeroTier { @@ -65,6 +66,12 @@ public: */ bool allowManaged; + /** + * Whitelist of addresses that can be configured by this network. + * If empty and allowManaged is true, allow all private/pseudoprivate addresses. + */ + std::vector allowManagedWhitelist; + /** * Allow configuration of IPs and routes within global (Internet) IP space? */ -- cgit v1.2.3 From 64774d0d4f552b2864abd969c6bc69c0ced3b2e1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 13:27:52 -0800 Subject: Replace piecemeal designation of upstreams with the concept of moons, which is simpler and easier to use and inherits all the cool live update stuff of worlds (now called planets) and global roots. --- include/ZeroTierOne.h | 15 +--- node/IncomingPacket.cpp | 22 ++--- node/Node.cpp | 85 ++++++-------------- node/Node.hpp | 1 - node/Peer.cpp | 12 +-- node/Switch.cpp | 2 +- node/Topology.cpp | 207 +++++++++++++++++++++++------------------------- node/Topology.hpp | 79 ++++++++++-------- node/World.hpp | 87 ++++++++++---------- service/OneService.cpp | 22 +++-- service/README.md | 1 - world/mkworld.cpp | 5 +- 12 files changed, 242 insertions(+), 296 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8b1ee0ac..f0235b9d 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -848,8 +848,8 @@ enum ZT_VirtualNetworkConfigOperation enum ZT_PeerRole { ZT_PEER_ROLE_LEAF = 0, // ordinary node - ZT_PEER_ROLE_UPSTREAM = 1, // upstream node - ZT_PEER_ROLE_ROOT = 2 // global root + ZT_PEER_ROLE_UPSTREAM = 1, // moon root + ZT_PEER_ROLE_ROOT = 2 // planetary root }; /** @@ -1903,17 +1903,6 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node); */ int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); -/** - * Set peer role - * - * Right now this can only be used to set a peer to either LEAF or - * UPSTREAM, since roots are fixed and defined by the World. - * - * @param ztAddress ZeroTier address (least significant 40 bits) - * @param role New peer role (LEAF or UPSTREAM) - */ -void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,enum ZT_PeerRole role); - /** * Set a network configuration master instance for this node * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 562aee91..2487a8aa 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -214,8 +214,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut Identity id; InetAddress externalSurfaceAddress; - uint64_t worldId = ZT_WORLD_ID_NULL; - uint64_t worldTimestamp = 0; + uint64_t planetWorldId = 0; + uint64_t planetWorldTimestamp = 0; { unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); @@ -223,10 +223,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut if (ptr < size()) ptr += externalSurfaceAddress.deserialize(*this,ptr); - // Get world ID and world timestamp if present (was not in old versions) + // Get primary planet world ID and world timestamp if present if ((ptr + 16) <= size()) { - worldId = at(ptr); ptr += 8; - worldTimestamp = at(ptr); + planetWorldId = at(ptr); ptr += 8; + planetWorldTimestamp = at(ptr); } } @@ -356,14 +356,14 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut tmpa.serialize(outp); } - if ((worldId != ZT_WORLD_ID_NULL)&&(RR->topology->worldTimestamp() > worldTimestamp)&&(worldId == RR->topology->worldId())) { - World w(RR->topology->world()); + if ((planetWorldId)&&(RR->topology->planetWorldTimestamp() > planetWorldTimestamp)&&(planetWorldId == RR->topology->planetWorldId())) { + World w(RR->topology->planet()); const unsigned int sizeAt = outp.size(); outp.addSize(2); // make room for 16-bit size field w.serialize(outp,false); outp.setAt(sizeAt,(uint16_t)(outp.size() - (sizeAt + 2))); } else { - outp.append((uint16_t)0); // no world update needed + outp.append((uint16_t)0); // no planet update needed } outp.armor(peer->key(),true); @@ -411,14 +411,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p if (ptr < size()) ptr += externalSurfaceAddress.deserialize(*this,ptr); - // Handle world updates from root servers if present (was not on old versions) - if (((ptr + 2) <= size())&&(RR->topology->isRoot(peer->identity()))) { + // Handle planet or moon updates + if ((ptr + 2) <= size()) { World worldUpdate; const unsigned int worldLen = at(ptr); ptr += 2; if (worldLen > 0) { World w; w.deserialize(*this,ptr); - RR->topology->worldUpdateIfValid(w); + RR->topology->addWorld(w,true); } } diff --git a/node/Node.cpp b/node/Node.cpp index 0d0750ca..df22e3f2 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -160,75 +160,48 @@ ZT_ResultCode Node::processVirtualNetworkFrame( class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,const std::vector
&upstreams,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), - _upstreams(upstreams), - _now(now), - _world(RR->topology->world()) + _now(now) { + RR->topology->getUpstreamStableEndpoints(_upstreams); } uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay inline void operator()(Topology &t,const SharedPtr &p) { - if (std::find(_upstreams.begin(),_upstreams.end(),p->address()) != _upstreams.end()) { - InetAddress stableEndpoint4,stableEndpoint6; - for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { - if (r->identity == p->identity()) { - for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)r->stableEndpoints.size();++k) { - const InetAddress &addr = r->stableEndpoints[ptr++ % r->stableEndpoints.size()]; - if (!stableEndpoint4) { - if (addr.ss_family == AF_INET) - stableEndpoint4 = addr; - } - if (!stableEndpoint6) { - if (addr.ss_family == AF_INET6) - stableEndpoint6 = addr; - } + const std::vector *upstreamStableEndpoints = _upstreams.get(p->address()); + if ((upstreamStableEndpoints)&&(upstreamStableEndpoints->size() > 0)) { + if (!p->doPingAndKeepalive(_now,AF_INET)) { + for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { + const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; + if (addr.ss_family == AF_INET) { + p->sendHELLO(InetAddress(),addr,_now); + break; } - break; } } - - // We keep connections to upstream peers alive forever. - bool needToContactIndirect = true; - if (p->doPingAndKeepalive(_now,AF_INET)) { - needToContactIndirect = false; - } else { - if (stableEndpoint4) { - needToContactIndirect = false; - p->sendHELLO(InetAddress(),stableEndpoint4,_now); - } - } - if (p->doPingAndKeepalive(_now,AF_INET6)) { - needToContactIndirect = false; - } else { - if (stableEndpoint6) { - needToContactIndirect = false; - p->sendHELLO(InetAddress(),stableEndpoint6,_now); + if (!p->doPingAndKeepalive(_now,AF_INET6)) { + for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { + const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; + if (addr.ss_family == AF_INET6) { + p->sendHELLO(InetAddress(),addr,_now); + break; + } } } - - // If we don't have a direct path or a static endpoint, send something indirectly to find one. - if (needToContactIndirect) { - Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP); - RR->sw->send(outp,true); - } - lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); } else if (p->isActive(_now)) { - // Normal nodes get their preferred link kept alive if the node has generated frame traffic recently p->doPingAndKeepalive(_now,-1); } } private: const RuntimeEnvironment *RR; - const std::vector
&_upstreams; uint64_t _now; - World _world; + Hashtable< Address,std::vector > _upstreams; }; ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) @@ -263,7 +236,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB } // Do pings and keepalives - _PingPeersThatNeedPing pfunc(RR,upstreams,now); + _PingPeersThatNeedPing pfunc(RR,now); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); // Update online status, post status change as event @@ -368,8 +341,8 @@ uint64_t Node::address() const void Node::status(ZT_NodeStatus *status) const { status->address = RR->identity.address().toInt(); - status->worldId = RR->topology->worldId(); - status->worldTimestamp = RR->topology->worldTimestamp(); + status->worldId = RR->topology->planetWorldId(); + status->worldTimestamp = RR->topology->planetWorldTimestamp(); status->publicIdentity = RR->publicIdentityStr.c_str(); status->secretIdentity = RR->secretIdentityStr.c_str(); status->relayPolicy = _relayPolicy; @@ -401,7 +374,7 @@ ZT_PeerList *Node::peers() const p->versionRev = -1; } p->latency = pi->second->latency(); - p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : (RR->topology->isUpstream(pi->second->identity()) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF); + p->role = RR->topology->role(pi->second->identity().address()); std::vector< std::pair< SharedPtr,bool > > paths(pi->second->paths(_now)); SharedPtr bestp(pi->second->getBestPath(_now,false)); @@ -488,11 +461,6 @@ int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigne return 0; } -void Node::setRole(uint64_t ztAddress,ZT_PeerRole role) -{ - RR->topology->setUpstream(Address(ztAddress),(role == ZT_PEER_ROLE_UPSTREAM)); -} - void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); @@ -1016,13 +984,6 @@ int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const vo } } -void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role) -{ - try { - reinterpret_cast(node)->setRole(ztAddress,role); - } catch ( ... ) {} -} - void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) { try { diff --git a/node/Node.hpp b/node/Node.hpp index d7b039b8..4c070014 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -105,7 +105,6 @@ public: int addLocalInterfaceAddress(const struct sockaddr_storage *addr); void clearLocalInterfaceAddresses(); int sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len); - void setRole(uint64_t ztAddress,ZT_PeerRole role); void setNetconfMaster(void *networkControllerInstance); ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); diff --git a/node/Peer.cpp b/node/Peer.cpp index 2ef139e1..40356034 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -37,9 +37,6 @@ namespace ZeroTier { -// Used to send varying values for NAT keepalive -static uint32_t _natKeepaliveBuf = 0; - Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : _lastReceive(0), _lastNontrivialReceive(0), @@ -355,8 +352,8 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u outp.append(now); RR->identity.serialize(outp,false); atAddress.serialize(outp); - outp.append((uint64_t)RR->topology->worldId()); - outp.append((uint64_t)RR->topology->worldTimestamp()); + outp.append((uint64_t)RR->topology->planetWorldId()); + outp.append((uint64_t)RR->topology->planetWorldTimestamp()); RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,false); // HELLO is sent in the clear RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); @@ -401,12 +398,9 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) } if (bestp >= 0) { - if ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) { + if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); _paths[bestp].path->sent(now); - } else if (_paths[bestp].path->needsHeartbeat(now)) { - _natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads - _paths[bestp].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now); } return true; } else { diff --git a/node/Switch.cpp b/node/Switch.cpp index 7c94d438..04624f03 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -718,7 +718,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) * go somewhere. */ SharedPtr viaPath(peer->getBestPath(now,false)); - if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) { + if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now); viaPath.zero(); diff --git a/node/Topology.cpp b/node/Topology.cpp index bf51b585..be6807da 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -48,33 +48,22 @@ Topology::Topology(const RuntimeEnvironment *renv) : _trustedPathCount(0), _amRoot(false) { - // Get cached world if present - std::string dsWorld(RR->node->dataStoreGet("world")); - World cachedWorld; - if (dsWorld.length() > 0) { - try { - Buffer dswtmp(dsWorld.data(),(unsigned int)dsWorld.length()); - cachedWorld.deserialize(dswtmp,0); - } catch ( ... ) { - cachedWorld = World(); // clear if cached world is invalid - } - } - - // Use default or cached world depending on which is shinier - World defaultWorld; + World defaultPlanet; { Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); - defaultWorld.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top + defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top } - if (cachedWorld.shouldBeReplacedBy(defaultWorld,false)) { - _setWorld(defaultWorld); - if (dsWorld.length() > 0) - RR->node->dataStoreDelete("world"); - } else _setWorld(cachedWorld); -} + addWorld(defaultPlanet,false); -Topology::~Topology() -{ + try { + World cachedPlanet; + std::string buf(RR->node->dataStoreGet("planet")); + if (buf.length() > 0) { + Buffer dswtmp(buf.data(),(unsigned int)buf.length()); + cachedPlanet.deserialize(dswtmp,0); + } + addWorld(cachedPlanet,false); + } catch ( ... ) {} } SharedPtr Topology::addPeer(const SharedPtr &peer) @@ -161,15 +150,14 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi Mutex::Lock _l(_lock); if (_amRoot) { - /* If I am a root server, the "best" root server is the one whose address - * is numerically greater than mine (with wrap at top of list). This - * causes packets searching for a route to pretty much literally - * circumnavigate the globe rather than bouncing between just two. */ - - for(unsigned long p=0;p<_rootAddresses.size();++p) { - if (_rootAddresses[p] == RR->identity.address()) { - for(unsigned long q=1;q<_rootAddresses.size();++q) { - const SharedPtr *const nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]); + /* If I am a root, pick another root that isn't mine and that + * has a numerically greater ID. This causes packets to roam + * around the top rather than bouncing between just two. */ + + for(unsigned long p=0;p<_upstreamAddresses.size();++p) { + if (_upstreamAddresses[p] == RR->identity.address()) { + for(unsigned long q=1;q<_upstreamAddresses.size();++q) { + const SharedPtr *const nextsn = _peers.get(_upstreamAddresses[(p + q) % _upstreamAddresses.size()]); if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) return *nextsn; } @@ -178,8 +166,7 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi } } else { - /* Otherwise pick the best upstream from among roots and any other - * designated upstreams that we trust. */ + /* Otherwise pick the bestest looking upstream */ unsigned int bestQualityOverall = ~((unsigned int)0); unsigned int bestQualityNotAvoid = ~((unsigned int)0); @@ -219,82 +206,112 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi return SharedPtr(); } -bool Topology::isRoot(const Identity &id) const -{ - Mutex::Lock _l(_lock); - return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end()); -} - bool Topology::isUpstream(const Identity &id) const { Mutex::Lock _l(_lock); return (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),id.address()) != _upstreamAddresses.end()); } -void Topology::setUpstream(const Address &a,bool upstream) +ZT_PeerRole Topology::role(const Address &ztaddr) const { - bool needWhois = false; - { - Mutex::Lock _l(_lock); - if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) { - if (upstream) { - if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) { - _upstreamAddresses.push_back(a); - const SharedPtr *p = _peers.get(a); - if (!p) { - const Identity id(_getIdentity(a)); - if (id) { - _peers.set(a,SharedPtr(new Peer(RR,RR->identity,id))); - } else { - needWhois = true; // need to do this later due to _lock - } - } - } - } else { - std::vector
ua; - for(std::vector
::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) { - if (a != *i) - ua.push_back(*i); - } - _upstreamAddresses.swap(ua); - } + Mutex::Lock _l(_lock); + if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) { + for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { + if (i->identity.address() == ztaddr) + return ZT_PEER_ROLE_ROOT; } + return ZT_PEER_ROLE_UPSTREAM; } - if (needWhois) - RR->sw->requestWhois(a); + return ZT_PEER_ROLE_LEAF; } bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const { Mutex::Lock _l(_lock); - if (std::find(_rootAddresses.begin(),_rootAddresses.end(),ztaddr) != _rootAddresses.end()) { - for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { + // For roots the only permitted addresses are those defined. This adds just a little + // bit of extra security against spoofing, replaying, etc. + if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) { + for(std::vector::const_iterator r(_planet.roots().begin());r!=_planet.roots().end();++r) { for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { if (ipaddr.ipsEqual(*e)) return false; } } + for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { + for(std::vector::const_iterator r(m->roots().begin());r!=m->roots().end();++r) { + for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { + if (ipaddr.ipsEqual(*e)) + return false; + } + } + } return true; } return false; } -bool Topology::worldUpdateIfValid(const World &newWorld) +bool Topology::addWorld(const World &newWorld,bool updateOnly) { + if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON)) + return false; + Mutex::Lock _l(_lock); - if (_world.shouldBeReplacedBy(newWorld,true)) { - _setWorld(newWorld); - try { - Buffer dswtmp; - newWorld.serialize(dswtmp,false); - RR->node->dataStorePut("world",dswtmp.data(),dswtmp.size(),false); - } catch ( ... ) { - RR->node->dataStoreDelete("world"); + + World *existing = (World *)0; + switch(newWorld.type()) { + case World::TYPE_PLANET: + existing = &_planet; + break; + case World::TYPE_MOON: + for(std::vector< World >::iterator m(_moons.begin());m!=_moons.end();++m) { + if (m->id() == newWorld.id()) { + existing = &(*m); + break; + } + } + break; + default: + return false; + } + + if (existing) { + if (existing->shouldBeReplacedBy(newWorld)) + *existing = newWorld; + else return false; + } else if ((newWorld.type() == World::TYPE_MOON)&&(!updateOnly)) { + _moons.push_back(newWorld); + existing = &(_moons.back()); + } else return false; + + char savePath[64]; + if (existing->type() == World::TYPE_MOON) + Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx",existing->id()); + else Utils::scopy(savePath,sizeof(savePath),"planet"); + try { + Buffer dswtmp; + existing->serialize(dswtmp,false); + RR->node->dataStorePut(savePath,dswtmp.data(),dswtmp.size(),false); + } catch ( ... ) { + RR->node->dataStoreDelete(savePath); + } + + _upstreamAddresses.clear(); + _amRoot = false; + for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { + if (i->identity == RR->identity) + _amRoot = true; + else _upstreamAddresses.push_back(i->identity.address()); + } + for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { + for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { + if (i->identity == RR->identity) + _amRoot = true; + else _upstreamAddresses.push_back(i->identity.address()); } - return true; } + return false; } @@ -334,34 +351,4 @@ Identity Topology::_getIdentity(const Address &zta) return Identity(); } -void Topology::_setWorld(const World &newWorld) -{ - // assumed _lock is locked (or in constructor) - - std::vector
ua; - for(std::vector
::iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { - if (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) - ua.push_back(*a); - } - - _world = newWorld; - _rootAddresses.clear(); - _amRoot = false; - - for(std::vector::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) { - _rootAddresses.push_back(r->identity.address()); - if (std::find(ua.begin(),ua.end(),r->identity.address()) == ua.end()) - ua.push_back(r->identity.address()); - if (r->identity.address() == RR->identity.address()) { - _amRoot = true; - } else { - SharedPtr *rp = _peers.get(r->identity.address()); - if (!rp) - _peers.set(r->identity.address(),SharedPtr(new Peer(RR,RR->identity,r->identity))); - } - } - - _upstreamAddresses.swap(ua); -} - } // namespace ZeroTier diff --git a/node/Topology.hpp b/node/Topology.hpp index 90ad7083..47981248 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -50,7 +50,6 @@ class Topology { public: Topology(const RuntimeEnvironment *renv); - ~Topology(); /** * Add a peer to database @@ -141,12 +140,6 @@ public: */ SharedPtr getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid); - /** - * @param id Identity to check - * @return True if this is a designated root server in this world - */ - bool isRoot(const Identity &id) const; - /** * @param id Identity to check * @return True if this is a root server or a network preferred relay from one of our networks @@ -154,14 +147,10 @@ public: bool isUpstream(const Identity &id) const; /** - * Set whether or not an address is upstream - * - * If the address is a root this does nothing, since roots are fixed. - * - * @param a Target address - * @param upstream New upstream status + * @param ztaddr ZeroTier address + * @return Peer role for this device */ - void setUpstream(const Address &a,bool upstream); + ZT_PeerRole role(const Address &ztaddr) const; /** * Check for prohibited endpoints @@ -179,6 +168,30 @@ public: */ bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const; + /** + * @param eps Hash table to fill with addresses and their stable endpoints + */ + inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector > &eps) const + { + Mutex::Lock _l(_lock); + for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { + std::vector &ips = eps[i->identity.address()]; + for(std::vector::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) { + if (std::find(ips.begin(),ips.end(),*j) == ips.end()) + ips.push_back(*j); + } + } + for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { + for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { + std::vector &ips = eps[i->identity.address()]; + for(std::vector::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) { + if (std::find(ips.begin(),ips.end(),*j) == ips.end()) + ips.push_back(*j); + } + } + } + } + /** * @return Vector of active upstream addresses (including roots) */ @@ -189,37 +202,38 @@ public: } /** - * @return Current World (copy) + * @return Current planet */ - inline World world() const + inline World planet() const { Mutex::Lock _l(_lock); - return _world; + return _planet; } /** - * @return Current world ID + * @return Current planet's world ID */ - inline uint64_t worldId() const + inline uint64_t planetWorldId() const { - return _world.id(); // safe to read without lock, and used from within eachPeer() so don't lock + return _planet.id(); // safe to read without lock, and used from within eachPeer() so don't lock } /** - * @return Current world timestamp + * @return Current planet's world timestamp */ - inline uint64_t worldTimestamp() const + inline uint64_t planetWorldTimestamp() const { - return _world.timestamp(); // safe to read without lock, and used from within eachPeer() so don't lock + return _planet.timestamp(); // safe to read without lock, and used from within eachPeer() so don't lock } /** * Validate new world and update if newer and signature is okay * - * @param newWorld Potential new world definition revision - * @return True if an update actually occurred + * @param newWorld A new or updated planet or moon to learn + * @param updateOnly If true only update currently known worlds + * @return True if it was valid and newer than current (or totally new for moons) */ - bool worldUpdateIfValid(const World &newWorld); + bool addWorld(const World &newWorld,bool updateOnly); /** * Clean and flush database @@ -284,9 +298,9 @@ public: } /** - * @return True if I am a root server in the current World + * @return True if I am a root server in a planet or moon */ - inline bool amRoot() const throw() { return _amRoot; } + inline bool amRoot() const { return _amRoot; } /** * Get the outbound trusted path ID for a physical address, or 0 if none @@ -339,7 +353,6 @@ public: private: Identity _getIdentity(const Address &zta); - void _setWorld(const World &newWorld); const RuntimeEnvironment *const RR; @@ -347,14 +360,14 @@ private: InetAddress _trustedPathNetworks[ZT_MAX_TRUSTED_PATHS]; unsigned int _trustedPathCount; - World _world; + World _planet; + std::vector< World > _moons; Hashtable< Address,SharedPtr > _peers; Hashtable< Path::HashKey,SharedPtr > _paths; - std::vector< Address > _upstreamAddresses; // includes roots - std::vector< Address > _rootAddresses; // only roots - bool _amRoot; // am I a root? + std::vector< Address > _upstreamAddresses; // includes root addresses of both planets and moons + bool _amRoot; // am I a root in a planet or moon? Mutex _lock; }; diff --git a/node/World.hpp b/node/World.hpp index 2f1edb00..c4682a69 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -48,16 +48,6 @@ */ #define ZT_WORLD_MAX_SERIALIZED_LENGTH (((1024 + (32 * ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT)) * ZT_WORLD_MAX_ROOTS) + ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_SIGNATURE_LEN + 128) -/** - * World ID indicating null / empty World object - */ -#define ZT_WORLD_ID_NULL 0 - -/** - * World ID for a test network with ephemeral or temporary roots - */ -#define ZT_WORLD_ID_TESTNET 1 - /** * World ID for Earth * @@ -90,15 +80,23 @@ namespace ZeroTier { * orbits, the Moon (about 1.3 light seconds), and nearby Lagrange points. A * world ID for Mars and nearby space is defined but not yet used, and a test * world ID is provided for testing purposes. - * - * If you absolutely must run your own "unofficial" ZeroTier network, please - * define your world IDs above 0xffffffff (4294967295). Code to make a World - * is in mkworld.cpp in the parent directory and must be edited to change - * settings. */ class World { public: + /** + * World type -- do not change IDs + */ + enum Type + { + TYPE_NULL = 0, + TYPE_PLANET = 1, // Planets, of which there is currently one (Earth) + TYPE_MOON = 127 // Moons, which are user-created and many + }; + + /** + * Upstream server definition in world/moon + */ struct Root { Identity identity; @@ -113,45 +111,44 @@ public: * Construct an empty / null World */ World() : - _id(ZT_WORLD_ID_NULL), - _ts(0) {} + _id(0), + _ts(0), + _type(TYPE_NULL) {} /** * @return Root servers for this world and their stable endpoints */ - inline const std::vector &roots() const throw() { return _roots; } + inline const std::vector &roots() const { return _roots; } + + /** + * @return World type: planet or moon + */ + inline Type type() const { return _type; } /** * @return World unique identifier */ - inline uint64_t id() const throw() { return _id; } + inline uint64_t id() const { return _id; } /** * @return World definition timestamp */ - inline uint64_t timestamp() const throw() { return _ts; } + inline uint64_t timestamp() const { return _ts; } /** * Check whether a world update should replace this one * - * A new world update is valid if it is for the same world ID, is newer, - * and is signed by the current world's signing key. If this world object - * is null, it can always be updated. - * * @param update Candidate update - * @param fullSignatureCheck Perform full cryptographic signature check (true == yes, false == skip) - * @return True if update is newer than current and is properly signed + * @return True if update is newer than current, matches its ID and type, and is properly signed (or if current is NULL) */ - inline bool shouldBeReplacedBy(const World &update,bool fullSignatureCheck) + inline bool shouldBeReplacedBy(const World &update) { - if (_id == ZT_WORLD_ID_NULL) + if ((_id == 0)||(_type == TYPE_NULL)) return true; - if ((_id == update._id)&&(_ts < update._ts)) { - if (fullSignatureCheck) { - Buffer tmp; - update.serialize(tmp,true); - return C25519::verify(_updatesMustBeSignedBy,tmp.data(),tmp.size(),update._signature); - } else return true; + if ((_id == update._id)&&(_ts < update._ts)&&(_type == update._type)) { + Buffer tmp; + update.serialize(tmp,true); + return C25519::verify(_updatesMustBeSignedBy,tmp.data(),tmp.size(),update._signature); } return false; } @@ -159,14 +156,14 @@ public: /** * @return True if this World is non-empty */ - inline operator bool() const throw() { return (_id != ZT_WORLD_ID_NULL); } + inline operator bool() const { return (_type != TYPE_NULL); } template inline void serialize(Buffer &b,bool forSign = false) const { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - b.append((uint8_t)0x01); + b.append((uint8_t)_type); b.append((uint64_t)_id); b.append((uint64_t)_ts); b.append(_updatesMustBeSignedBy.data,ZT_C25519_PUBLIC_KEY_LEN); @@ -190,14 +187,19 @@ public: _roots.clear(); - if (b[p++] != 0x01) - throw std::invalid_argument("invalid object type"); + switch((Type)b[p++]) { + case TYPE_NULL: _type = TYPE_NULL; break; // shouldn't ever really happen in serialized data but it's not invalid + case TYPE_PLANET: _type = TYPE_PLANET; break; + case TYPE_MOON: _type = TYPE_MOON; break; + default: + throw std::invalid_argument("invalid world type"); + } _id = b.template at(p); p += 8; _ts = b.template at(p); p += 8; memcpy(_updatesMustBeSignedBy.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN; memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; - unsigned int numRoots = b[p++]; + const unsigned int numRoots = (unsigned int)b[p++]; if (numRoots > ZT_WORLD_MAX_ROOTS) throw std::invalid_argument("too many roots in World"); for(unsigned int k=0;k _roots; diff --git a/service/OneService.cpp b/service/OneService.cpp index 2932c605..6d9effa1 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -422,7 +422,7 @@ public: try { std::string authToken; { - std::string authTokenPath(_homePath + ZT_PATH_SEPARATOR_S + "authtoken.secret"); + std::string authTokenPath(_homePath + ZT_PATH_SEPARATOR_S "authtoken.secret"); if (!OSUtils::readFile(authTokenPath.c_str(),authToken)) { unsigned char foo[24]; Utils::getSecureRandom(foo,sizeof(foo)); @@ -442,7 +442,8 @@ public: authToken = _trimString(authToken); // Clean up any legacy files if present - OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + "peers.save").c_str()); + OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S "peers.save").c_str()); + OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S "world").c_str()); { struct ZT_Node_Callbacks cb; @@ -465,7 +466,7 @@ public: unsigned int trustedPathCount = 0; // Old style "trustedpaths" flat file -- will eventually go away - FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S + "trustedpaths").c_str(),"r"); + FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r"); if (trustpaths) { char buf[1024]; while ((fgets(buf,sizeof(buf),trustpaths))&&(trustedPathCount < ZT_MAX_TRUSTED_PATHS)) { @@ -493,7 +494,7 @@ public: // Read local config file Mutex::Lock _l2(_localConfig_m); std::string lcbuf; - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "local.conf").c_str(),lcbuf)) { + if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(),lcbuf)) { try { _localConfig = OSUtils::jsonParse(lcbuf); if (!_localConfig.is_object()) { @@ -581,7 +582,7 @@ public: // Write file containing primary port to be read by CLIs, etc. char portstr[64]; Utils::snprintf(portstr,sizeof(portstr),"%u",_ports[0]); - OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "zerotier-one.port").c_str(),std::string(portstr)); + OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr)); // Attempt to bind to a secondary port chosen from our ZeroTier address. // This exists because there are buggy NATs out there that fail if more @@ -641,8 +642,8 @@ public: _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER - if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str())) { - _clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str()); + if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S "cluster").c_str())) { + _clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S "cluster").c_str()); if (_clusterDefinition->size() > 0) { std::vector members(_clusterDefinition->members()); for(std::vector::iterator m(members.begin());m!=members.end();++m) { @@ -689,12 +690,12 @@ public: } #endif - _controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str()); + _controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S "ui").c_str()); _controlPlane->addAuthToken(authToken.c_str()); _controlPlane->setController(_controller); { // Remember networks from previous session - std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str())); + std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str())); for(std::vector::iterator f(networksDotD.begin());f!=networksDotD.end();++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16)&&(f->substr(16) == ".conf")) @@ -919,9 +920,6 @@ public: if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) { const Address ztaddr(nstr.c_str()); if (ztaddr) { - const std::string rstr(OSUtils::jsonString(v.value()["role"],"")); - _node->setRole(ztaddr.toInt(),((rstr == "upstream")||(rstr == "UPSTREAM")) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF); - const uint64_t ztaddr2 = ztaddr.toInt(); std::vector &v4h = _v4Hints[ztaddr2]; std::vector &v6h = _v6Hints[ztaddr2]; diff --git a/service/README.md b/service/README.md index d2398643..5d54b923 100644 --- a/service/README.md +++ b/service/README.md @@ -19,7 +19,6 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al }, "virtual": { /* Settings applied to ZeroTier virtual network devices (VL1) */ "##########": { /* 10-digit ZeroTier address */ - "role": "upstream"|"leaf", /* If upstream, define this as a trusted "federated root" (default is leaf) */ "try": [ "IP/port"/*,...*/ ], /* Hints on where to reach this peer if no upstreams/roots are online */ "blacklist": [ "NETWORK/bits"/*,...*/ ] /* Blacklist a physical path for only this peer. */ } diff --git a/world/mkworld.cpp b/world/mkworld.cpp index 061d6341..2e9e621f 100644 --- a/world/mkworld.cpp +++ b/world/mkworld.cpp @@ -53,11 +53,12 @@ using namespace ZeroTier; class WorldMaker : public World { public: - static inline World make(uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector &roots,const C25519::Pair &signWith) + static inline World make(World::Type t,uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector &roots,const C25519::Pair &signWith) { WorldMaker w; w._id = id; w._ts = ts; + w._type = t; w._updateSigningKey = sk; w._roots = roots; @@ -139,7 +140,7 @@ int main(int argc,char **argv) fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts); - World nw = WorldMaker::make(id,ts,currentKP.pub,roots,previousKP); + World nw = WorldMaker::make(World::TYPE_PLANET,id,ts,currentKP.pub,roots,previousKP); Buffer outtmp; nw.serialize(outtmp,false); -- cgit v1.2.3 From f102fd7f92225fbe39ae69dda716530a4e5457e9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 13:50:56 -0800 Subject: Extend in-band world updates to handle moons too. --- node/IncomingPacket.cpp | 40 ++++++++++++++++++++++++++++++---------- node/Packet.hpp | 13 +++++++++---- node/Peer.cpp | 11 +++++++++++ node/Topology.hpp | 9 +++++++++ node/World.hpp | 4 ++++ 5 files changed, 63 insertions(+), 14 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 2487a8aa..1a60d13a 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -216,6 +216,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut InetAddress externalSurfaceAddress; uint64_t planetWorldId = 0; uint64_t planetWorldTimestamp = 0; + std::vector< std::pair > moonIdsAndTimestamps; { unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); @@ -228,6 +229,16 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut planetWorldId = at(ptr); ptr += 8; planetWorldTimestamp = at(ptr); } + + // Get moon IDs and timestamps if present + if ((ptr + 2) <= size()) { + unsigned int numMoons = at(ptr); ptr += 2; + for(unsigned int i=0;i(at(ptr),at(ptr + 8))); + ptr += 16; + } + } } if (fromAddress != id.address()) { @@ -356,15 +367,24 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut tmpa.serialize(outp); } + const unsigned int worldUpdateSizeAt = outp.size(); + outp.addSize(2); // make room for 16-bit size field if ((planetWorldId)&&(RR->topology->planetWorldTimestamp() > planetWorldTimestamp)&&(planetWorldId == RR->topology->planetWorldId())) { - World w(RR->topology->planet()); - const unsigned int sizeAt = outp.size(); - outp.addSize(2); // make room for 16-bit size field - w.serialize(outp,false); - outp.setAt(sizeAt,(uint16_t)(outp.size() - (sizeAt + 2))); - } else { - outp.append((uint16_t)0); // no planet update needed + RR->topology->planet().serialize(outp,false); + } + if (moonIdsAndTimestamps.size() > 0) { + std::vector moons(RR->topology->moons()); + for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { + for(std::vector< std::pair >::const_iterator i(moonIdsAndTimestamps.begin());i!=moonIdsAndTimestamps.end();++i) { + if (i->first == m->id()) { + if (m->timestamp() > i->second) + m->serialize(outp,false); + break; + } + } + } } + outp.setAt(worldUpdateSizeAt,(uint16_t)(outp.size() - (worldUpdateSizeAt + 2))); outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),now); @@ -411,11 +431,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p if (ptr < size()) ptr += externalSurfaceAddress.deserialize(*this,ptr); - // Handle planet or moon updates + // Handle planet or moon updates if present (older versions don't send this) if ((ptr + 2) <= size()) { - World worldUpdate; const unsigned int worldLen = at(ptr); ptr += 2; - if (worldLen > 0) { + const unsigned int endOfWorlds = ptr + worldLen; + while (ptr < endOfWorlds) { World w; w.deserialize(*this,ptr); RR->topology->addWorld(w,true); diff --git a/node/Packet.hpp b/node/Packet.hpp index 0be19f8a..26e87af8 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -536,12 +536,17 @@ public: * <[1] software major version> * <[1] software minor version> * <[2] software revision> - * <[8] timestamp (ms since epoch)> + * <[8] timestamp for determining latench> * <[...] binary serialized identity (see Identity)> * <[1] destination address type> * [<[...] destination address to which packet was sent>] - * <[8] 64-bit world ID of current world> - * <[8] 64-bit timestamp of current world> + * <[8] 64-bit world ID of current planet> + * <[8] 64-bit timestamp of current planet> + * <[2] 16-bit number of moons> + * [<[1] 8-bit type ID of moon>] + * [<[8] 64-bit world ID of moon>] + * [<[8] 64-bit timestamp of moon>] + * [... additional moons ...] * * This is the only message that ever must be sent in the clear, since it * is used to push an identity to a new peer. @@ -567,7 +572,7 @@ public: * <[1] destination address type (for this OK, not copied from HELLO)> * [<[...] destination address>] * <[2] 16-bit length of world update or 0 if none> - * [[...] world update] + * [[...] updates to planets and/or moons] * * ERROR has no payload. */ diff --git a/node/Peer.cpp b/node/Peer.cpp index 40356034..441a5b33 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -345,6 +345,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); + outp.append((unsigned char)ZT_PROTO_VERSION); outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR); outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR); @@ -352,8 +353,18 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u outp.append(now); RR->identity.serialize(outp,false); atAddress.serialize(outp); + outp.append((uint64_t)RR->topology->planetWorldId()); outp.append((uint64_t)RR->topology->planetWorldTimestamp()); + + std::vector moons(RR->topology->moons()); + outp.append((uint16_t)moons.size()); + for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { + outp.append((uint8_t)m->type()); + outp.append((uint64_t)m->id()); + outp.append((uint64_t)m->timestamp()); + } + RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,false); // HELLO is sent in the clear RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); diff --git a/node/Topology.hpp b/node/Topology.hpp index 47981248..6369c5cd 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -201,6 +201,15 @@ public: return _upstreamAddresses; } + /** + * @return Current moons + */ + inline std::vector moons() const + { + Mutex::Lock _l(_lock); + return _moons; + } + /** * @return Current planet */ diff --git a/node/World.hpp b/node/World.hpp index c4682a69..06dcb981 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -176,6 +176,8 @@ public: for(std::vector::const_iterator ep(r->stableEndpoints.begin());ep!=r->stableEndpoints.end();++ep) ep->serialize(b); } + if (_type == TYPE_MOON) + b.append((uint16_t)0); // no attached dictionary (for future use) if (forSign) b.append((uint64_t)0xf7f7f7f7f7f7f7f7ULL); } @@ -214,6 +216,8 @@ public: p += r.stableEndpoints.back().deserialize(b,p); } } + if (_type == TYPE_MOON) + p += b.template at(p) + 2; return (p - startAt); } -- cgit v1.2.3 From bc218f9414bd6e4124eb223c7c69c5ac254befff Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 13:52:29 -0800 Subject: little fix --- node/IncomingPacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 1a60d13a..28b845b4 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -437,7 +437,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const unsigned int endOfWorlds = ptr + worldLen; while (ptr < endOfWorlds) { World w; - w.deserialize(*this,ptr); + ptr += w.deserialize(*this,ptr); RR->topology->addWorld(w,true); } } -- cgit v1.2.3 From 0b3b994241161c996c8432a2fb25e47b0f84c359 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 14:05:09 -0800 Subject: Relay policy can now be computed. --- include/ZeroTierOne.h | 24 ------------------------ node/Network.cpp | 1 - node/NetworkConfig.hpp | 2 -- node/Node.cpp | 22 +--------------------- node/Node.hpp | 3 --- node/Switch.cpp | 27 +++++---------------------- service/ControlPlane.cpp | 2 -- service/OneService.cpp | 9 +-------- service/README.md | 1 - 9 files changed, 7 insertions(+), 84 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index f0235b9d..f75638f8 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -406,16 +406,6 @@ enum ZT_Event ZT_EVENT_USER_MESSAGE = 6 }; -/** - * Node relay policy - */ -enum ZT_RelayPolicy -{ - ZT_RELAY_POLICY_NEVER = 0, - ZT_RELAY_POLICY_TRUSTED = 1, - ZT_RELAY_POLICY_ALWAYS = 2 -}; - /** * User message used with ZT_EVENT_USER_MESSAGE */ @@ -476,11 +466,6 @@ typedef struct */ const char *secretIdentity; - /** - * Node relay policy - */ - enum ZT_RelayPolicy relayPolicy; - /** * True if some kind of connectivity appears available */ @@ -1718,15 +1703,6 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( */ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); -/** - * Set node's relay policy - * - * @param node Node instance - * @param rp New relay policy - * @return OK(0) or error code - */ -enum ZT_ResultCode ZT_Node_setRelayPolicy(ZT_Node *node,enum ZT_RelayPolicy rp); - /** * Join a network * diff --git a/node/Network.cpp b/node/Network.cpp index 8b0f2055..ec1bcb33 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1102,7 +1102,6 @@ void Network::requestConfiguration() rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); - rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY,(uint64_t)RR->node->relayPolicy()); if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index a548e866..39087395 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -135,8 +135,6 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH "a" // Network configuration meta-data flags #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS "f" -// Relay policy for this node -#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY "rp" // These dictionary keys are short so they don't take up much room. // By convention we use upper case for binary blobs, but it doesn't really matter. diff --git a/node/Node.cpp b/node/Node.cpp index df22e3f2..23271cca 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -53,8 +53,7 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : _prngStreamPtr(0), _now(now), _lastPingCheck(0), - _lastHousekeepingRun(0), - _relayPolicy(ZT_RELAY_POLICY_TRUSTED) + _lastHousekeepingRun(0) { if (callbacks->version != 0) throw std::runtime_error("callbacks struct version mismatch"); @@ -102,9 +101,6 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : throw; } - if (RR->topology->amRoot()) - _relayPolicy = ZT_RELAY_POLICY_ALWAYS; - postEvent(ZT_EVENT_UP); } @@ -282,12 +278,6 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB return ZT_RESULT_OK; } -ZT_ResultCode Node::setRelayPolicy(enum ZT_RelayPolicy rp) -{ - _relayPolicy = rp; - return ZT_RESULT_OK; -} - ZT_ResultCode Node::join(uint64_t nwid,void *uptr) { Mutex::Lock _l(_networks_m); @@ -345,7 +335,6 @@ void Node::status(ZT_NodeStatus *status) const status->worldTimestamp = RR->topology->planetWorldTimestamp(); status->publicIdentity = RR->publicIdentityStr.c_str(); status->secretIdentity = RR->secretIdentityStr.c_str(); - status->relayPolicy = _relayPolicy; status->online = _online ? 1 : 0; } @@ -860,15 +849,6 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol } } -enum ZT_ResultCode ZT_Node_setRelayPolicy(ZT_Node *node,enum ZT_RelayPolicy rp) -{ - try { - return reinterpret_cast(node)->setRelayPolicy(rp); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr) { try { diff --git a/node/Node.hpp b/node/Node.hpp index 4c070014..662abcb4 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -91,7 +91,6 @@ public: unsigned int frameLength, volatile uint64_t *nextBackgroundTaskDeadline); ZT_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode setRelayPolicy(enum ZT_RelayPolicy rp); ZT_ResultCode join(uint64_t nwid,void *uptr); ZT_ResultCode leave(uint64_t nwid,void **uptr); ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); @@ -197,7 +196,6 @@ public: inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,nwid,nuptr,op,nc); } inline bool online() const throw() { return _online; } - inline ZT_RelayPolicy relayPolicy() const { return _relayPolicy; } #ifdef ZT_TRACE void postTrace(const char *module,unsigned int line,const char *fmt,...); @@ -298,7 +296,6 @@ private: uint64_t _now; uint64_t _lastPingCheck; uint64_t _lastHousekeepingRun; - ZT_RelayPolicy _relayPolicy; bool _online; }; diff --git a/node/Switch.cpp b/node/Switch.cpp index 04624f03..f935b7aa 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -105,17 +105,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from const Address destination(fragment.destination()); if (destination != RR->identity.address()) { - switch(RR->node->relayPolicy()) { - case ZT_RELAY_POLICY_ALWAYS: - break; - case ZT_RELAY_POLICY_TRUSTED: - if (!path->trustEstablished(now)) - return; - break; - // case ZT_RELAY_POLICY_NEVER: - default: - return; - } + if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) + return; if (fragment.hops() < ZT_RELAY_MAX_HOPS) { fragment.incrementHops(); @@ -213,18 +204,10 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); + if (destination != RR->identity.address()) { - switch(RR->node->relayPolicy()) { - case ZT_RELAY_POLICY_ALWAYS: - break; - case ZT_RELAY_POLICY_TRUSTED: - if (!path->trustEstablished(now)) - return; - break; - // case ZT_RELAY_POLICY_NEVER: - default: - return; - } + if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) + return; Packet packet(data,len); diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 86158a91..e5142f3e 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -391,7 +391,6 @@ unsigned int ControlPlane::handleRequest( "\t\"worldId\": %llu,\n" "\t\"worldTimestamp\": %llu,\n" "\t\"online\": %s,\n" - "\t\"relayPolicy\": \"%s\",\n" "\t\"tcpFallbackActive\": %s,\n" "\t\"versionMajor\": %d,\n" "\t\"versionMinor\": %d,\n" @@ -405,7 +404,6 @@ unsigned int ControlPlane::handleRequest( status.worldId, status.worldTimestamp, (status.online) ? "true" : "false", - ((status.relayPolicy == ZT_RELAY_POLICY_ALWAYS) ? "always" : ((status.relayPolicy == ZT_RELAY_POLICY_NEVER) ? "never" : "trusted")), (_svc->tcpFallbackActive()) ? "true" : "false", ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, diff --git a/service/OneService.cpp b/service/OneService.cpp index 6d9effa1..f6174d42 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -694,7 +694,7 @@ public: _controlPlane->addAuthToken(authToken.c_str()); _controlPlane->setController(_controller); - { // Remember networks from previous session + { // Load existing networks std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str())); for(std::vector::iterator f(networksDotD.begin());f!=networksDotD.end();++f) { std::size_t dot = f->find_last_of('.'); @@ -981,13 +981,6 @@ public: if (settings.is_object()) { _portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"],true); - const std::string rp(OSUtils::jsonString(settings["relayPolicy"],"")); - if ((rp == "always")||(rp == "ALWAYS")) - _node->setRelayPolicy(ZT_RELAY_POLICY_ALWAYS); - else if ((rp == "never")||(rp == "NEVER")) - _node->setRelayPolicy(ZT_RELAY_POLICY_NEVER); - else _node->setRelayPolicy(ZT_RELAY_POLICY_TRUSTED); - const std::string up(OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT)); const bool udist = OSUtils::jsonBool(settings["softwareUpdateDist"],false); if (((up == "apply")||(up == "download"))||(udist)) { diff --git a/service/README.md b/service/README.md index 5d54b923..d3bc5338 100644 --- a/service/README.md +++ b/service/README.md @@ -24,7 +24,6 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al } }, "settings": { /* Other global settings */ - "relayPolicy": "trusted"|"always"|"never", /* Policy for relaying others' traffic (see below) */ "portMappingEnabled": true|false, /* If true (the default), try to use uPnP or NAT-PMP to map ports */ "softwareUpdate": "apply"|"download"|"disable", /* Automatically apply updates, just download, or disable built-in software updates */ "softwareUpdateDist": true|false, /* If true, distribute software updates (only really useful to ZeroTier, Inc. itself, default is false) */ -- cgit v1.2.3 From 9f7919f71f6b4326e73759923d2cf747affc0244 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 15:27:26 -0800 Subject: Add comments to join ("orbit") moons. --- include/ZeroTierOne.h | 22 +++++++++ node/IncomingPacket.cpp | 3 +- node/Node.cpp | 49 ++++++++++++++++++-- node/Node.hpp | 2 + node/Peer.cpp | 9 +++- node/Topology.cpp | 120 ++++++++++++++++++++++++++++++++++++++++-------- node/Topology.hpp | 39 ++++++++++++++-- service/OneService.cpp | 8 ++++ 8 files changed, 222 insertions(+), 30 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index f75638f8..6c50a0a6 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1779,6 +1779,28 @@ enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64 */ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); +/** + * Add or update a moon + * + * Moons are persisted in the data store in moons.d/, so this can persist + * across invocations if the contents of moon.d are scanned and orbit is + * called for each on startup. + * + * @param moonWorldId Moon's world ID + * @param len Length of moonWorld in bytes + * @return Error if moon was invalid or failed to be added + */ +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId); + +/** + * Remove a moon (does nothing if not present) + * + * @param node Node instance + * @param moonWorldId World ID of moon to remove + * @return Error if anything bad happened + */ +ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId); + /** * Get this node's 40-bit ZeroTier address * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 28b845b4..93bf4590 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -444,7 +444,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); - peer->addDirectLatencyMeasurment(latency); + if (!hops()) + peer->addDirectLatencyMeasurment(latency); peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if ((externalSurfaceAddress)&&(hops() == 0)) diff --git a/node/Node.cpp b/node/Node.cpp index 23271cca..f5ee1f9d 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -168,26 +168,35 @@ public: inline void operator()(Topology &t,const SharedPtr &p) { - const std::vector *upstreamStableEndpoints = _upstreams.get(p->address()); - if ((upstreamStableEndpoints)&&(upstreamStableEndpoints->size() > 0)) { + const std::vector *const upstreamStableEndpoints = _upstreams.get(p->address()); + if (upstreamStableEndpoints) { + bool contacted = false; + if (!p->doPingAndKeepalive(_now,AF_INET)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET) { p->sendHELLO(InetAddress(),addr,_now); + contacted = true; break; } } - } + } else contacted = true; + if (!p->doPingAndKeepalive(_now,AF_INET6)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET6) { p->sendHELLO(InetAddress(),addr,_now); + contacted = true; break; } } - } + } else contacted = true; + + if (!contacted) + p->sendHELLO(InetAddress(),InetAddress(),_now); + lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); } else if (p->isActive(_now)) { p->doPingAndKeepalive(_now,-1); @@ -224,7 +233,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) (*n)->requestConfiguration(); - // Run WHOIS on upstreams we don't know about + // Attempt to get identity for any unknown upstreams const std::vector
upstreams(RR->topology->upstreamAddresses()); for(std::vector
::const_iterator a(upstreams.begin());a!=upstreams.end();++a) { if (!RR->topology->getPeer(*a)) @@ -323,6 +332,18 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } +ZT_ResultCode Node::orbit(uint64_t moonWorldId) +{ + RR->topology->addMoon(moonWorldId); + return ZT_RESULT_OK; +} + +ZT_ResultCode Node::deorbit(uint64_t moonWorldId) +{ + RR->topology->removeMoon(moonWorldId); + return ZT_RESULT_OK; +} + uint64_t Node::address() const { return RR->identity.address().toInt(); @@ -893,6 +914,24 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint } } +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId) +{ + try { + return reinterpret_cast(node)->orbit(moonWorldId); + } catch ( ... ) { + return ZT_RESULT_FATAL_ERROR_INTERNAL; + } +} + +ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId) +{ + try { + return reinterpret_cast(node)->deorbit(moonWorldId); + } catch ( ... ) { + return ZT_RESULT_FATAL_ERROR_INTERNAL; + } +} + uint64_t ZT_Node_address(ZT_Node *node) { return reinterpret_cast(node)->address(); diff --git a/node/Node.hpp b/node/Node.hpp index 662abcb4..3e742092 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -95,6 +95,8 @@ public: ZT_ResultCode leave(uint64_t nwid,void **uptr); ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); + ZT_ResultCode orbit(uint64_t moonWorldId); + ZT_ResultCode deorbit(uint64_t moonWorldId); uint64_t address() const; void status(ZT_NodeStatus *status) const; ZT_PeerList *peers() const; diff --git a/node/Peer.cpp b/node/Peer.cpp index 441a5b33..50135b9f 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -366,8 +366,13 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u } RR->node->expectReplyTo(outp.packetId()); - outp.armor(_key,false); // HELLO is sent in the clear - RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + + if (atAddress) { + outp.armor(_key,false); + RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + } else { + RR->sw->send(outp,false); + } } void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now) diff --git a/node/Topology.cpp b/node/Topology.cpp index be6807da..38afacb0 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -48,13 +48,6 @@ Topology::Topology(const RuntimeEnvironment *renv) : _trustedPathCount(0), _amRoot(false) { - World defaultPlanet; - { - Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); - defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top - } - addWorld(defaultPlanet,false); - try { World cachedPlanet; std::string buf(RR->node->dataStoreGet("planet")); @@ -64,6 +57,13 @@ Topology::Topology(const RuntimeEnvironment *renv) : } addWorld(cachedPlanet,false); } catch ( ... ) {} + + World defaultPlanet; + { + Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); + defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top + } + addWorld(defaultPlanet,false); } SharedPtr Topology::addPeer(const SharedPtr &peer) @@ -287,7 +287,7 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly) char savePath[64]; if (existing->type() == World::TYPE_MOON) - Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx",existing->id()); + Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",existing->id()); else Utils::scopy(savePath,sizeof(savePath),"planet"); try { Buffer dswtmp; @@ -297,22 +297,71 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly) RR->node->dataStoreDelete(savePath); } - _upstreamAddresses.clear(); - _amRoot = false; - for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { - if (i->identity == RR->identity) - _amRoot = true; - else _upstreamAddresses.push_back(i->identity.address()); + if (existing->type() == World::TYPE_MOON) { + std::vector
cm; + for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) { + if (m->toInt() != ((existing->id() >> 24) & 0xffffffffffULL)) + cm.push_back(*m); + } + _contacingMoons.swap(cm); } + + _memoizeUpstreams(); + + return true; +} + +void Topology::addMoon(const uint64_t id) +{ + char savePath[64]; + Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); + + try { + std::string moonBin(RR->node->dataStoreGet(savePath)); + if (moonBin.length() > 1) { + Buffer wtmp(moonBin.data(),(unsigned int)moonBin.length()); + World w; + w.deserialize(wtmp); + if (w.type() == World::TYPE_MOON) { + addWorld(w,false); + return; + } + } + } catch ( ... ) {} + + { + const Address a(id >> 24); + Mutex::Lock _l(_lock); + if (std::find(_contacingMoons.begin(),_contacingMoons.end(),a) == _contacingMoons.end()) + _contacingMoons.push_back(a); + } + RR->node->dataStorePut(savePath,"\0",1,false); // persist that we want to be a member +} + +void Topology::removeMoon(const uint64_t id) +{ + Mutex::Lock _l(_lock); + + std::vector nm; for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { - for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { - if (i->identity == RR->identity) - _amRoot = true; - else _upstreamAddresses.push_back(i->identity.address()); + if (m->id() != id) { + nm.push_back(*m); + } else { + char savePath[64]; + Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); + RR->node->dataStoreDelete(savePath); } } + _moons.swap(nm); - return false; + std::vector
cm; + for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) { + if (m->toInt() != ((id >> 24) & 0xffffffffffULL)) + cm.push_back(*m); + } + _contacingMoons.swap(cm); + + _memoizeUpstreams(); } void Topology::clean(uint64_t now) @@ -351,4 +400,37 @@ Identity Topology::_getIdentity(const Address &zta) return Identity(); } +void Topology::_memoizeUpstreams() +{ + // assumes _lock is locked + _upstreamAddresses.clear(); + _amRoot = false; + for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { + if (i->identity == RR->identity) { + _amRoot = true; + } else { + _upstreamAddresses.push_back(i->identity.address()); + SharedPtr &hp = _peers[i->identity.address()]; + if (!hp) { + hp = new Peer(RR,RR->identity,i->identity); + saveIdentity(i->identity); + } + } + } + for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { + for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { + if (i->identity == RR->identity) { + _amRoot = true; + } else { + _upstreamAddresses.push_back(i->identity.address()); + SharedPtr &hp = _peers[i->identity.address()]; + if (!hp) { + hp = new Peer(RR,RR->identity,i->identity); + saveIdentity(i->identity); + } + } + } + } +} + } // namespace ZeroTier diff --git a/node/Topology.hpp b/node/Topology.hpp index 6369c5cd..693ae12c 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -169,6 +169,11 @@ public: bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const; /** + * This gets the known stable endpoints for any upstream + * + * It also adds empty entries for any upstreams we are attempting to + * contact. + * * @param eps Hash table to fill with addresses and their stable endpoints */ inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector > &eps) const @@ -190,6 +195,8 @@ public: } } } + for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) + eps[*m]; } /** @@ -198,7 +205,12 @@ public: inline std::vector
upstreamAddresses() const { Mutex::Lock _l(_lock); - return _upstreamAddresses; + std::vector
u(_upstreamAddresses); + for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) { + if (std::find(u.begin(),u.end(),*m) == u.end()) + u.push_back(*m); + } + return u; } /** @@ -244,6 +256,25 @@ public: */ bool addWorld(const World &newWorld,bool updateOnly); + /** + * Add a moon + * + * This loads it from moons.d if present, and if not adds it to + * a list of moons that we want to contact. It does not actually + * send anything, though this will happen on the next background + * task loop where pings etc. are checked. + * + * @param id Moon ID + */ + void addMoon(const uint64_t id); + + /** + * Remove a moon + * + * @param id Moon's world ID + */ + void removeMoon(const uint64_t id); + /** * Clean and flush database */ @@ -362,6 +393,7 @@ public: private: Identity _getIdentity(const Address &zta); + void _memoizeUpstreams(); const RuntimeEnvironment *const RR; @@ -375,8 +407,9 @@ private: Hashtable< Address,SharedPtr > _peers; Hashtable< Path::HashKey,SharedPtr > _paths; - std::vector< Address > _upstreamAddresses; // includes root addresses of both planets and moons - bool _amRoot; // am I a root in a planet or moon? + std::vector
_contacingMoons; + std::vector
_upstreamAddresses; + bool _amRoot; Mutex _lock; }; diff --git a/service/OneService.cpp b/service/OneService.cpp index f6174d42..d2ebe6b7 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -702,6 +702,14 @@ public: _node->join(Utils::hexStrToU64(f->substr(0,dot).c_str()),(void *)0); } } + { // Load existing moons + std::vector moonsDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "moons.d").c_str())); + for(std::vector::iterator f(moonsDotD.begin());f!=moonsDotD.end();++f) { + std::size_t dot = f->find_last_of('.'); + if ((dot == 16)&&(f->substr(16) == ".moon")) + _node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str())); + } + } _nextBackgroundTaskDeadline = 0; uint64_t clockShouldBe = OSUtils::now(); -- cgit v1.2.3 From 1d775af34a5efa6008256d1bfa742c28ee7152ab Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 15:35:21 -0800 Subject: Fix moon persistence. --- node/IncomingPacket.cpp | 2 +- node/Node.cpp | 2 +- node/Topology.cpp | 43 +++++++++++++++++++++---------------------- node/Topology.hpp | 9 ++++----- 4 files changed, 27 insertions(+), 29 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 93bf4590..a3fbbefc 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -438,7 +438,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p while (ptr < endOfWorlds) { World w; ptr += w.deserialize(*this,ptr); - RR->topology->addWorld(w,true); + RR->topology->addWorld(w); } } diff --git a/node/Node.cpp b/node/Node.cpp index f5ee1f9d..c4a40395 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -220,7 +220,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB try { _lastPingCheck = now; - // Get relays and networks that need config without leaving the mutex locked + // Get networks that need config without leaving mutex locked std::vector< SharedPtr > needConfig; { Mutex::Lock _l(_networks_m); diff --git a/node/Topology.cpp b/node/Topology.cpp index 38afacb0..ece93ee6 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -55,7 +55,7 @@ Topology::Topology(const RuntimeEnvironment *renv) : Buffer dswtmp(buf.data(),(unsigned int)buf.length()); cachedPlanet.deserialize(dswtmp,0); } - addWorld(cachedPlanet,false); + addWorld(cachedPlanet); } catch ( ... ) {} World defaultPlanet; @@ -63,7 +63,7 @@ Topology::Topology(const RuntimeEnvironment *renv) : Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top } - addWorld(defaultPlanet,false); + addWorld(defaultPlanet); } SharedPtr Topology::addPeer(const SharedPtr &peer) @@ -252,7 +252,7 @@ bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipa return false; } -bool Topology::addWorld(const World &newWorld,bool updateOnly) +bool Topology::addWorld(const World &newWorld) { if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON)) return false; @@ -280,9 +280,16 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly) if (existing->shouldBeReplacedBy(newWorld)) *existing = newWorld; else return false; - } else if ((newWorld.type() == World::TYPE_MOON)&&(!updateOnly)) { + } else if ((newWorld.type() == World::TYPE_MOON)&&(std::find(_contactingMoons.begin(),_contactingMoons.end(),Address(newWorld.id() >> 24)) != _contactingMoons.end())) { _moons.push_back(newWorld); existing = &(_moons.back()); + + std::vector
cm; + for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { + if (m->toInt() != ((existing->id() >> 24) & 0xffffffffffULL)) + cm.push_back(*m); + } + _contactingMoons.swap(cm); } else return false; char savePath[64]; @@ -297,15 +304,6 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly) RR->node->dataStoreDelete(savePath); } - if (existing->type() == World::TYPE_MOON) { - std::vector
cm; - for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) { - if (m->toInt() != ((existing->id() >> 24) & 0xffffffffffULL)) - cm.push_back(*m); - } - _contacingMoons.swap(cm); - } - _memoizeUpstreams(); return true; @@ -313,6 +311,13 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly) void Topology::addMoon(const uint64_t id) { + { + const Address a(id >> 24); + Mutex::Lock _l(_lock); + if (std::find(_contactingMoons.begin(),_contactingMoons.end(),a) == _contactingMoons.end()) + _contactingMoons.push_back(a); + } + char savePath[64]; Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); @@ -323,18 +328,12 @@ void Topology::addMoon(const uint64_t id) World w; w.deserialize(wtmp); if (w.type() == World::TYPE_MOON) { - addWorld(w,false); + addWorld(w); return; } } } catch ( ... ) {} - { - const Address a(id >> 24); - Mutex::Lock _l(_lock); - if (std::find(_contacingMoons.begin(),_contacingMoons.end(),a) == _contacingMoons.end()) - _contacingMoons.push_back(a); - } RR->node->dataStorePut(savePath,"\0",1,false); // persist that we want to be a member } @@ -355,11 +354,11 @@ void Topology::removeMoon(const uint64_t id) _moons.swap(nm); std::vector
cm; - for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) { + for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { if (m->toInt() != ((id >> 24) & 0xffffffffffULL)) cm.push_back(*m); } - _contacingMoons.swap(cm); + _contactingMoons.swap(cm); _memoizeUpstreams(); } diff --git a/node/Topology.hpp b/node/Topology.hpp index 693ae12c..e8efe0db 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -195,7 +195,7 @@ public: } } } - for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) + for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) eps[*m]; } @@ -206,7 +206,7 @@ public: { Mutex::Lock _l(_lock); std::vector
u(_upstreamAddresses); - for(std::vector
::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) { + for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { if (std::find(u.begin(),u.end(),*m) == u.end()) u.push_back(*m); } @@ -251,10 +251,9 @@ public: * Validate new world and update if newer and signature is okay * * @param newWorld A new or updated planet or moon to learn - * @param updateOnly If true only update currently known worlds * @return True if it was valid and newer than current (or totally new for moons) */ - bool addWorld(const World &newWorld,bool updateOnly); + bool addWorld(const World &newWorld); /** * Add a moon @@ -407,7 +406,7 @@ private: Hashtable< Address,SharedPtr > _peers; Hashtable< Path::HashKey,SharedPtr > _paths; - std::vector
_contacingMoons; + std::vector
_contactingMoons; std::vector
_upstreamAddresses; bool _amRoot; -- cgit v1.2.3 From 9e7c778cc8ebdfd5d3f773a7d3cb30ad154e7189 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 16:16:06 -0800 Subject: Fix deadlock. --- node/Node.cpp | 13 +++++++++---- node/Topology.cpp | 31 ++++++++++++++++++------------- node/Topology.hpp | 39 +++++++++++++++++---------------------- 3 files changed, 44 insertions(+), 39 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index c4a40395..3d5b5c3d 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -159,7 +159,8 @@ public: _PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), - _now(now) + _now(now), + _bestCurrentUpstream(RR->topology->getUpstreamPeer()) { RR->topology->getUpstreamStableEndpoints(_upstreams); } @@ -194,8 +195,11 @@ public: } } else contacted = true; - if (!contacted) - p->sendHELLO(InetAddress(),InetAddress(),_now); + if ((!contacted)&&(_bestCurrentUpstream)) { + const SharedPtr up(_bestCurrentUpstream->getBestPath(_now,true)); + if (up) + p->sendHELLO(up->localAddress(),up->address(),_now); + } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); } else if (p->isActive(_now)) { @@ -205,7 +209,8 @@ public: private: const RuntimeEnvironment *RR; - uint64_t _now; + const uint64_t _now; + const SharedPtr _bestCurrentUpstream; Hashtable< Address,std::vector > _upstreams; }; diff --git a/node/Topology.cpp b/node/Topology.cpp index ece93ee6..5632c337 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -79,7 +79,7 @@ SharedPtr Topology::addPeer(const SharedPtr &peer) SharedPtr np; { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); SharedPtr &hp = _peers[peer->address()]; if (!hp) hp = peer; @@ -99,7 +99,7 @@ SharedPtr Topology::getPeer(const Address &zta) } { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); const SharedPtr *const ap = _peers.get(zta); if (ap) return *ap; @@ -110,7 +110,7 @@ SharedPtr Topology::getPeer(const Address &zta) if (id) { SharedPtr np(new Peer(RR,RR->identity,id)); { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); SharedPtr &ap = _peers[zta]; if (!ap) ap.swap(np); @@ -127,7 +127,7 @@ Identity Topology::getIdentity(const Address &zta) if (zta == RR->identity.address()) { return RR->identity; } else { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); const SharedPtr *const ap = _peers.get(zta); if (ap) return (*ap)->identity(); @@ -147,7 +147,8 @@ void Topology::saveIdentity(const Identity &id) SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid) { const uint64_t now = RR->node->now(); - Mutex::Lock _l(_lock); + Mutex::Lock _l1(_peers_m); + Mutex::Lock _l2(_upstreams_m); if (_amRoot) { /* If I am a root, pick another root that isn't mine and that @@ -208,13 +209,13 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi bool Topology::isUpstream(const Identity &id) const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); return (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),id.address()) != _upstreamAddresses.end()); } ZT_PeerRole Topology::role(const Address &ztaddr) const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) { for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { if (i->identity.address() == ztaddr) @@ -227,7 +228,7 @@ ZT_PeerRole Topology::role(const Address &ztaddr) const bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); // For roots the only permitted addresses are those defined. This adds just a little // bit of extra security against spoofing, replaying, etc. @@ -257,7 +258,8 @@ bool Topology::addWorld(const World &newWorld) if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON)) return false; - Mutex::Lock _l(_lock); + Mutex::Lock _l1(_upstreams_m); + Mutex::Lock _l2(_peers_m); World *existing = (World *)0; switch(newWorld.type()) { @@ -313,7 +315,7 @@ void Topology::addMoon(const uint64_t id) { { const Address a(id >> 24); - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); if (std::find(_contactingMoons.begin(),_contactingMoons.end(),a) == _contactingMoons.end()) _contactingMoons.push_back(a); } @@ -339,7 +341,8 @@ void Topology::addMoon(const uint64_t id) void Topology::removeMoon(const uint64_t id) { - Mutex::Lock _l(_lock); + Mutex::Lock _l1(_upstreams_m); + Mutex::Lock _l2(_peers_m); std::vector nm; for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { @@ -365,8 +368,9 @@ void Topology::removeMoon(const uint64_t id) void Topology::clean(uint64_t now) { - Mutex::Lock _l(_lock); { + Mutex::Lock _l1(_peers_m); + Mutex::Lock _l2(_upstreams_m); Hashtable< Address,SharedPtr >::Iterator i(_peers); Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; @@ -376,6 +380,7 @@ void Topology::clean(uint64_t now) } } { + Mutex::Lock _l(_paths_m); Hashtable< Path::HashKey,SharedPtr >::Iterator i(_paths); Path::HashKey *k = (Path::HashKey *)0; SharedPtr *p = (SharedPtr *)0; @@ -401,7 +406,7 @@ Identity Topology::_getIdentity(const Address &zta) void Topology::_memoizeUpstreams() { - // assumes _lock is locked + // assumes _upstreams_m and _peers_m are locked _upstreamAddresses.clear(); _amRoot = false; for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { diff --git a/node/Topology.hpp b/node/Topology.hpp index e8efe0db..78dc0fe8 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -82,7 +82,7 @@ public: */ inline SharedPtr getPeerNoCache(const Address &zta) { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); const SharedPtr *const ap = _peers.get(zta); if (ap) return *ap; @@ -98,7 +98,7 @@ public: */ inline SharedPtr getPath(const InetAddress &l,const InetAddress &r) { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_paths_m); SharedPtr &p = _paths[Path::HashKey(l,r)]; if (!p) p.setToUnsafe(new Path(l,r)); @@ -178,7 +178,7 @@ public: */ inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector > &eps) const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { std::vector &ips = eps[i->identity.address()]; for(std::vector::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) { @@ -204,7 +204,7 @@ public: */ inline std::vector
upstreamAddresses() const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); std::vector
u(_upstreamAddresses); for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { if (std::find(u.begin(),u.end(),*m) == u.end()) @@ -218,7 +218,7 @@ public: */ inline std::vector moons() const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); return _moons; } @@ -227,7 +227,7 @@ public: */ inline World planet() const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_upstreams_m); return _planet; } @@ -286,7 +286,7 @@ public: inline unsigned long countActive(uint64_t now) const { unsigned long cnt = 0; - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); Hashtable< Address,SharedPtr >::Iterator i(const_cast(this)->_peers); Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; @@ -299,20 +299,13 @@ public: /** * Apply a function or function object to all peers * - * Note: explicitly template this by reference if you want the object - * passed by reference instead of copied. - * - * Warning: be careful not to use features in these that call any other - * methods of Topology that may lock _lock, otherwise a recursive lock - * and deadlock or lock corruption may occur. - * * @param f Function to apply * @tparam F Function or function object type */ template inline void eachPeer(F f) { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); Hashtable< Address,SharedPtr >::Iterator i(_peers); Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; @@ -332,7 +325,7 @@ public: */ inline std::vector< std::pair< Address,SharedPtr > > allPeers() const { - Mutex::Lock _l(_lock); + Mutex::Lock _l(_peers_m); return _peers.entries(); } @@ -382,7 +375,7 @@ public: { if (count > ZT_MAX_TRUSTED_PATHS) count = ZT_MAX_TRUSTED_PATHS; - Mutex::Lock _l(_lock); + Mutex::Lock _l(_trustedPaths_m); for(unsigned int i=0;i _moons; + Mutex _trustedPaths_m; Hashtable< Address,SharedPtr > _peers; + Mutex _peers_m; + Hashtable< Path::HashKey,SharedPtr > _paths; + Mutex _paths_m; + World _planet; + std::vector _moons; std::vector
_contactingMoons; std::vector
_upstreamAddresses; bool _amRoot; - - Mutex _lock; + Mutex _upstreams_m; // locks worlds, upstream info, moon info, etc. }; } // namespace ZeroTier -- cgit v1.2.3 From 77a1dd4737cfaccfd013f85a5feaa786c58a9a35 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 16:25:53 -0800 Subject: Dead code removal, fix minor issue in upstream endpoint check. --- node/Topology.cpp | 20 ++++++++++++++------ one.cpp | 29 ----------------------------- 2 files changed, 14 insertions(+), 35 deletions(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index 5632c337..0cd3db9e 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -234,16 +234,24 @@ bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipa // bit of extra security against spoofing, replaying, etc. if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) { for(std::vector::const_iterator r(_planet.roots().begin());r!=_planet.roots().end();++r) { - for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { - if (ipaddr.ipsEqual(*e)) - return false; + if (r->identity.address() == ztaddr) { + if (r->stableEndpoints.size() == 0) + return false; // no stable endpoints specified, so allow dynamic paths + for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { + if (ipaddr.ipsEqual(*e)) + return false; + } } } for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { for(std::vector::const_iterator r(m->roots().begin());r!=m->roots().end();++r) { - for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { - if (ipaddr.ipsEqual(*e)) - return false; + if (r->identity.address() == ztaddr) { + if (r->stableEndpoints.size() == 0) + return false; // no stable endpoints specified, so allow dynamic paths + for(std::vector::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) { + if (ipaddr.ipsEqual(*e)) + return false; + } } } } diff --git a/one.cpp b/one.cpp index 43af7dea..016aab74 100644 --- a/one.cpp +++ b/one.cpp @@ -545,7 +545,6 @@ static void idtoolPrintHelp(FILE *out,const char *pn) fprintf(out," getpublic " ZT_EOL_S); fprintf(out," sign " ZT_EOL_S); fprintf(out," verify " ZT_EOL_S); - fprintf(out," mkcom [ ...] (hexadecimal integers)" ZT_EOL_S); } static Identity getIdFromArg(char *arg) @@ -690,34 +689,6 @@ static int idtool(int argc,char **argv) fprintf(stderr,"%s signature check FAILED" ZT_EOL_S,argv[3]); return 1; } - } else if (!strcmp(argv[1],"mkcom")) { - if (argc < 3) { - idtoolPrintHelp(stdout,argv[0]); - return 1; - } - - Identity id = getIdFromArg(argv[2]); - if ((!id)||(!id.hasPrivate())) { - fprintf(stderr,"Identity argument invalid, does not include private key, or file unreadable: %s" ZT_EOL_S,argv[2]); - return 1; - } - - CertificateOfMembership com; - for(int a=3;a params(OSUtils::split(argv[a],",","","")); - if (params.size() == 3) { - uint64_t qId = Utils::hexStrToU64(params[0].c_str()); - uint64_t qValue = Utils::hexStrToU64(params[1].c_str()); - uint64_t qMaxDelta = Utils::hexStrToU64(params[2].c_str()); - com.setQualifier(qId,qValue,qMaxDelta); - } - } - if (!com.sign(id)) { - fprintf(stderr,"Signature of certificate of membership failed." ZT_EOL_S); - return 1; - } - - printf("%s",com.toString().c_str()); } else { idtoolPrintHelp(stdout,argv[0]); return 1; -- cgit v1.2.3 From 5fa1d9796cbed473bdac1f1a4b84d5f6e5a1c69c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 27 Jan 2017 17:34:39 -0800 Subject: zerotier-idtool commands to init and generate moons --- attic/world/README.md | 7 ++ attic/world/build.sh | 1 + attic/world/earth-2016-01-13.bin | Bin 0 -> 634 bytes attic/world/mkworld.cpp | 149 ++++++++++++++++++++++++++++++ attic/world/old/earth-2015-11-16.bin | Bin 0 -> 494 bytes attic/world/old/earth-2015-11-20.bin | Bin 0 -> 792 bytes attic/world/old/earth-2015-12-17.bin | Bin 0 -> 732 bytes node/World.hpp | 27 ++++++ one.cpp | 87 ++++++++++++++++++ world/README.md | 7 -- world/build.sh | 1 - world/earth-2016-01-13.bin | Bin 634 -> 0 bytes world/mkworld.cpp | 169 ----------------------------------- world/old/earth-2015-11-16.bin | Bin 494 -> 0 bytes world/old/earth-2015-11-20.bin | Bin 792 -> 0 bytes world/old/earth-2015-12-17.bin | Bin 732 -> 0 bytes 16 files changed, 271 insertions(+), 177 deletions(-) create mode 100644 attic/world/README.md create mode 100755 attic/world/build.sh create mode 100644 attic/world/earth-2016-01-13.bin create mode 100644 attic/world/mkworld.cpp create mode 100644 attic/world/old/earth-2015-11-16.bin create mode 100644 attic/world/old/earth-2015-11-20.bin create mode 100644 attic/world/old/earth-2015-12-17.bin delete mode 100644 world/README.md delete mode 100755 world/build.sh delete mode 100644 world/earth-2016-01-13.bin delete mode 100644 world/mkworld.cpp delete mode 100644 world/old/earth-2015-11-16.bin delete mode 100644 world/old/earth-2015-11-20.bin delete mode 100644 world/old/earth-2015-12-17.bin (limited to 'node') diff --git a/attic/world/README.md b/attic/world/README.md new file mode 100644 index 00000000..dda4920a --- /dev/null +++ b/attic/world/README.md @@ -0,0 +1,7 @@ +World Definitions and Generator Code +====== + +This little bit of code is used to generate world updates. Ordinary users probably will never need this unless they want to test or experiment. + +See mkworld.cpp for documentation. To build from this directory use 'source ./build.sh'. + diff --git a/attic/world/build.sh b/attic/world/build.sh new file mode 100755 index 00000000..b783702c --- /dev/null +++ b/attic/world/build.sh @@ -0,0 +1 @@ +c++ -I.. -o mkworld ../node/C25519.cpp ../node/Salsa20.cpp ../node/SHA512.cpp ../node/Identity.cpp ../node/Utils.cpp ../node/InetAddress.cpp ../osdep/OSUtils.cpp mkworld.cpp diff --git a/attic/world/earth-2016-01-13.bin b/attic/world/earth-2016-01-13.bin new file mode 100644 index 00000000..5dea4d21 Binary files /dev/null and b/attic/world/earth-2016-01-13.bin differ diff --git a/attic/world/mkworld.cpp b/attic/world/mkworld.cpp new file mode 100644 index 00000000..e0f477b3 --- /dev/null +++ b/attic/world/mkworld.cpp @@ -0,0 +1,149 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * This utility makes the World from the configuration specified below. + * It probably won't be much use to anyone outside ZeroTier, Inc. except + * for testing and experimentation purposes. + * + * If you want to make your own World you must edit this file. + * + * When run, it expects two files in the current directory: + * + * previous.c25519 - key pair to sign this world (key from previous world) + * current.c25519 - key pair whose public key should be embedded in this world + * + * If these files do not exist, they are both created with the same key pair + * and a self-signed initial World is born. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace ZeroTier; + +int main(int argc,char **argv) +{ + std::string previous,current; + if ((!OSUtils::readFile("previous.c25519",previous))||(!OSUtils::readFile("current.c25519",current))) { + C25519::Pair np(C25519::generate()); + previous = std::string(); + previous.append((const char *)np.pub.data,ZT_C25519_PUBLIC_KEY_LEN); + previous.append((const char *)np.priv.data,ZT_C25519_PRIVATE_KEY_LEN); + current = previous; + OSUtils::writeFile("previous.c25519",previous); + OSUtils::writeFile("current.c25519",current); + fprintf(stderr,"INFO: created initial world keys: previous.c25519 and current.c25519 (both initially the same)"ZT_EOL_S); + } + + if ((previous.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))||(current.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))) { + fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid"ZT_EOL_S); + return 1; + } + C25519::Pair previousKP; + memcpy(previousKP.pub.data,previous.data(),ZT_C25519_PUBLIC_KEY_LEN); + memcpy(previousKP.priv.data,previous.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN); + C25519::Pair currentKP; + memcpy(currentKP.pub.data,current.data(),ZT_C25519_PUBLIC_KEY_LEN); + memcpy(currentKP.priv.data,current.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN); + + // ========================================================================= + // EDIT BELOW HERE + + std::vector roots; + + const uint64_t id = ZT_WORLD_ID_EARTH; + const uint64_t ts = 1452708876314ULL; // January 13th, 2016 + + // Alice + roots.push_back(World::Root()); + roots.back().identity = Identity("9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e"); + roots.back().stableEndpoints.push_back(InetAddress("188.166.94.177/9993")); // Amsterdam + roots.back().stableEndpoints.push_back(InetAddress("2a03:b0c0:2:d0::7d:1/9993")); // Amsterdam + roots.back().stableEndpoints.push_back(InetAddress("154.66.197.33/9993")); // Johannesburg + roots.back().stableEndpoints.push_back(InetAddress("2c0f:f850:154:197::33/9993")); // Johannesburg + roots.back().stableEndpoints.push_back(InetAddress("159.203.97.171/9993")); // New York + roots.back().stableEndpoints.push_back(InetAddress("2604:a880:800:a1::54:6001/9993")); // New York + roots.back().stableEndpoints.push_back(InetAddress("169.57.143.104/9993")); // Sao Paolo + roots.back().stableEndpoints.push_back(InetAddress("2607:f0d0:1d01:57::2/9993")); // Sao Paolo + roots.back().stableEndpoints.push_back(InetAddress("107.170.197.14/9993")); // San Francisco + roots.back().stableEndpoints.push_back(InetAddress("2604:a880:1:20::200:e001/9993")); // San Francisco + roots.back().stableEndpoints.push_back(InetAddress("128.199.197.217/9993")); // Singapore + roots.back().stableEndpoints.push_back(InetAddress("2400:6180:0:d0::b7:4001/9993")); // Singapore + + // Bob + roots.push_back(World::Root()); + roots.back().identity = Identity("8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c"); + roots.back().stableEndpoints.push_back(InetAddress("45.32.198.130/9993")); // Dallas + roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:6400:81c3:5400:00ff:fe18:1d61/9993")); // Dallas + roots.back().stableEndpoints.push_back(InetAddress("46.101.160.249/9993")); // Frankfurt + roots.back().stableEndpoints.push_back(InetAddress("2a03:b0c0:3:d0::6a:3001/9993")); // Frankfurt + roots.back().stableEndpoints.push_back(InetAddress("107.191.46.210/9993")); // Paris + roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:6800:83a4::64/9993")); // Paris + roots.back().stableEndpoints.push_back(InetAddress("45.32.246.179/9993")); // Sydney + roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:5800:8bf8:5400:ff:fe15:b39a/9993")); // Sydney + roots.back().stableEndpoints.push_back(InetAddress("45.32.248.87/9993")); // Tokyo + roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:7000:9bc9:5400:00ff:fe15:c4f5/9993")); // Tokyo + roots.back().stableEndpoints.push_back(InetAddress("159.203.2.154/9993")); // Toronto + roots.back().stableEndpoints.push_back(InetAddress("2604:a880:cad:d0::26:7001/9993")); // Toronto + + // END WORLD DEFINITION + // ========================================================================= + + fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts); + + World nw = World::make(World::TYPE_PLANET,id,ts,currentKP.pub,roots,previousKP); + + Buffer outtmp; + nw.serialize(outtmp,false); + World testw; + testw.deserialize(outtmp,0); + if (testw != nw) { + fprintf(stderr,"FATAL: serialization test failed!"ZT_EOL_S); + return 1; + } + + OSUtils::writeFile("world.bin",std::string((const char *)outtmp.data(),outtmp.size())); + fprintf(stderr,"INFO: world.bin written with %u bytes of binary world data."ZT_EOL_S,outtmp.size()); + + fprintf(stdout,ZT_EOL_S); + fprintf(stdout,"#define ZT_DEFAULT_WORLD_LENGTH %u"ZT_EOL_S,outtmp.size()); + fprintf(stdout,"static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {"); + for(unsigned int i=0;i 0) + fprintf(stdout,","); + fprintf(stdout,"0x%.2x",(unsigned int)d[i]); + } + fprintf(stdout,"};"ZT_EOL_S); + + return 0; +} diff --git a/attic/world/old/earth-2015-11-16.bin b/attic/world/old/earth-2015-11-16.bin new file mode 100644 index 00000000..910ff144 Binary files /dev/null and b/attic/world/old/earth-2015-11-16.bin differ diff --git a/attic/world/old/earth-2015-11-20.bin b/attic/world/old/earth-2015-11-20.bin new file mode 100644 index 00000000..198682e5 Binary files /dev/null and b/attic/world/old/earth-2015-11-20.bin differ diff --git a/attic/world/old/earth-2015-12-17.bin b/attic/world/old/earth-2015-12-17.bin new file mode 100644 index 00000000..20fadb56 Binary files /dev/null and b/attic/world/old/earth-2015-12-17.bin differ diff --git a/node/World.hpp b/node/World.hpp index 06dcb981..8fe6dd2e 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -227,6 +227,33 @@ public: inline bool operator<(const World &w) const { return (((int)_type < (int)w._type) ? true : ((_type == w._type) ? (_id < w._id) : false)); } + /** + * Create a World object signed with a key pair + * + * @param t World type + * @param id World ID + * @param ts World timestamp / revision + * @param sk Key that must be used to sign the next future update to this world + * @param roots Roots and their stable endpoints + * @param signWith Key to sign this World with (can have the same public as the next-update signing key, but doesn't have to) + * @return Signed World object + */ + static inline World make(World::Type t,uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector &roots,const C25519::Pair &signWith) + { + World w; + w._id = id; + w._ts = ts; + w._type = t; + w._updatesMustBeSignedBy = sk; + w._roots = roots; + + Buffer tmp; + w.serialize(tmp,true); + w._signature = C25519::sign(signWith,tmp.data(),tmp.size()); + + return w; + } + protected: uint64_t _id; uint64_t _ts; diff --git a/one.cpp b/one.cpp index 016aab74..37ea13d3 100644 --- a/one.cpp +++ b/one.cpp @@ -62,6 +62,8 @@ #include "node/CertificateOfMembership.hpp" #include "node/Utils.hpp" #include "node/NetworkController.hpp" +#include "node/Buffer.hpp" +#include "node/World.hpp" #include "osdep/OSUtils.hpp" #include "osdep/Http.hpp" @@ -545,6 +547,8 @@ static void idtoolPrintHelp(FILE *out,const char *pn) fprintf(out," getpublic " ZT_EOL_S); fprintf(out," sign " ZT_EOL_S); fprintf(out," verify " ZT_EOL_S); + fprintf(out," initmoon " ZT_EOL_S); + fprintf(out," genmoon " ZT_EOL_S); } static Identity getIdFromArg(char *arg) @@ -689,6 +693,89 @@ static int idtool(int argc,char **argv) fprintf(stderr,"%s signature check FAILED" ZT_EOL_S,argv[3]); return 1; } + } else if (!strcmp(argv[1],"initmoon")) { + if (argc < 3) { + idtoolPrintHelp(stdout,argv[0]); + } else { + const Identity id = getIdFromArg(argv[2]); + if (!id) { + fprintf(stderr,"%s is not a valid identity" ZT_EOL_S,argv[2]); + return 1; + } + + C25519::Pair kp(C25519::generate()); + + nlohmann::json mj; + mj["objtype"] = "world"; + mj["worldType"] = "moon"; + mj["updatesMustBeSignedBy"] = mj["signingKey"] = Utils::hex(kp.pub.data,(unsigned int)kp.pub.size()); + mj["signingKeySECRET"] = Utils::hex(kp.priv.data,(unsigned int)kp.priv.size()); + mj["id"] = (id.address().toString() + "000000"); + nlohmann::json seedj; + seedj["identity"] = id.toString(false); + seedj["stableEndpoints"] = nlohmann::json::array(); + (mj["roots"] = nlohmann::json::array()).push_back(seedj); + std::string mjd(OSUtils::jsonDump(mj)); + + printf("%s" ZT_EOL_S,mjd.c_str()); + } + } else if (!strcmp(argv[1],"genmoon")) { + if (argc < 3) { + idtoolPrintHelp(stdout,argv[0]); + } else { + std::string buf; + if (!OSUtils::readFile(argv[2],buf)) { + fprintf(stderr,"cannot read %s" ZT_EOL_S,argv[2]); + return 1; + } + nlohmann::json mj(OSUtils::jsonParse(buf)); + + uint64_t id = Utils::hexStrToU64(OSUtils::jsonString(mj["id"],"").c_str()); + + World::Type t; + if (mj["worldType"] == "moon") { + t = World::TYPE_MOON; + } else if (mj["worldType"] == "planet") { + t = World::TYPE_PLANET; + } else { + fprintf(stderr,"invalid worldType" ZT_EOL_S); + return 1; + } + + C25519::Pair signingKey; + C25519::Public updatesMustBeSignedBy; + Utils::unhex(OSUtils::jsonString(mj["singingKey"],""),signingKey.pub.data,(unsigned int)signingKey.pub.size()); + Utils::unhex(OSUtils::jsonString(mj["singingKeySECRET"],""),signingKey.priv.data,(unsigned int)signingKey.priv.size()); + Utils::unhex(OSUtils::jsonString(mj["updatesMustBeSignedBy"],""),updatesMustBeSignedBy.data,(unsigned int)updatesMustBeSignedBy.size()); + + std::vector roots; + nlohmann::json &rootsj = mj["roots"]; + if (rootsj.is_array()) { + for(unsigned long i=0;i<(unsigned long)rootsj.size();++i) { + nlohmann::json &r = rootsj[i]; + if (r.is_object()) { + roots.push_back(World::Root()); + roots.back().identity = Identity(OSUtils::jsonString(r["identity"],"")); + nlohmann::json &stableEndpointsj = r["stableEndpoints"]; + if (stableEndpointsj.is_array()) { + for(unsigned long k=0;k<(unsigned long)stableEndpointsj.size();++k) + roots.back().stableEndpoints.push_back(InetAddress(OSUtils::jsonString(stableEndpointsj[k],""))); + std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end()); + } + } + } + } + std::sort(roots.begin(),roots.end()); + + const uint64_t now = OSUtils::now(); + World w(World::make(t,id,now,updatesMustBeSignedBy,roots,signingKey)); + Buffer wbuf; + w.serialize(wbuf); + char fn[128]; + Utils::snprintf(fn,sizeof(fn),"%.16llx_%.16llx.moon",w.id(),now); + OSUtils::writeFile(fn,wbuf.data(),wbuf.size()); + printf("wrote %s (signed world with timestamp %llu)" ZT_EOL_S,fn,now); + } } else { idtoolPrintHelp(stdout,argv[0]); return 1; diff --git a/world/README.md b/world/README.md deleted file mode 100644 index dda4920a..00000000 --- a/world/README.md +++ /dev/null @@ -1,7 +0,0 @@ -World Definitions and Generator Code -====== - -This little bit of code is used to generate world updates. Ordinary users probably will never need this unless they want to test or experiment. - -See mkworld.cpp for documentation. To build from this directory use 'source ./build.sh'. - diff --git a/world/build.sh b/world/build.sh deleted file mode 100755 index b783702c..00000000 --- a/world/build.sh +++ /dev/null @@ -1 +0,0 @@ -c++ -I.. -o mkworld ../node/C25519.cpp ../node/Salsa20.cpp ../node/SHA512.cpp ../node/Identity.cpp ../node/Utils.cpp ../node/InetAddress.cpp ../osdep/OSUtils.cpp mkworld.cpp diff --git a/world/earth-2016-01-13.bin b/world/earth-2016-01-13.bin deleted file mode 100644 index 5dea4d21..00000000 Binary files a/world/earth-2016-01-13.bin and /dev/null differ diff --git a/world/mkworld.cpp b/world/mkworld.cpp deleted file mode 100644 index 2e9e621f..00000000 --- a/world/mkworld.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -/* - * This utility makes the World from the configuration specified below. - * It probably won't be much use to anyone outside ZeroTier, Inc. except - * for testing and experimentation purposes. - * - * If you want to make your own World you must edit this file. - * - * When run, it expects two files in the current directory: - * - * previous.c25519 - key pair to sign this world (key from previous world) - * current.c25519 - key pair whose public key should be embedded in this world - * - * If these files do not exist, they are both created with the same key pair - * and a self-signed initial World is born. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -using namespace ZeroTier; - -class WorldMaker : public World -{ -public: - static inline World make(World::Type t,uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector &roots,const C25519::Pair &signWith) - { - WorldMaker w; - w._id = id; - w._ts = ts; - w._type = t; - w._updateSigningKey = sk; - w._roots = roots; - - Buffer tmp; - w.serialize(tmp,true); - w._signature = C25519::sign(signWith,tmp.data(),tmp.size()); - - return w; - } -}; - -int main(int argc,char **argv) -{ - std::string previous,current; - if ((!OSUtils::readFile("previous.c25519",previous))||(!OSUtils::readFile("current.c25519",current))) { - C25519::Pair np(C25519::generate()); - previous = std::string(); - previous.append((const char *)np.pub.data,ZT_C25519_PUBLIC_KEY_LEN); - previous.append((const char *)np.priv.data,ZT_C25519_PRIVATE_KEY_LEN); - current = previous; - OSUtils::writeFile("previous.c25519",previous); - OSUtils::writeFile("current.c25519",current); - fprintf(stderr,"INFO: created initial world keys: previous.c25519 and current.c25519 (both initially the same)"ZT_EOL_S); - } - - if ((previous.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))||(current.length() != (ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_PRIVATE_KEY_LEN))) { - fprintf(stderr,"FATAL: previous.c25519 or current.c25519 empty or invalid"ZT_EOL_S); - return 1; - } - C25519::Pair previousKP; - memcpy(previousKP.pub.data,previous.data(),ZT_C25519_PUBLIC_KEY_LEN); - memcpy(previousKP.priv.data,previous.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN); - C25519::Pair currentKP; - memcpy(currentKP.pub.data,current.data(),ZT_C25519_PUBLIC_KEY_LEN); - memcpy(currentKP.priv.data,current.data() + ZT_C25519_PUBLIC_KEY_LEN,ZT_C25519_PRIVATE_KEY_LEN); - - // ========================================================================= - // EDIT BELOW HERE - - std::vector roots; - - const uint64_t id = ZT_WORLD_ID_EARTH; - const uint64_t ts = 1452708876314ULL; // January 13th, 2016 - - // Alice - roots.push_back(World::Root()); - roots.back().identity = Identity("9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e"); - roots.back().stableEndpoints.push_back(InetAddress("188.166.94.177/9993")); // Amsterdam - roots.back().stableEndpoints.push_back(InetAddress("2a03:b0c0:2:d0::7d:1/9993")); // Amsterdam - roots.back().stableEndpoints.push_back(InetAddress("154.66.197.33/9993")); // Johannesburg - roots.back().stableEndpoints.push_back(InetAddress("2c0f:f850:154:197::33/9993")); // Johannesburg - roots.back().stableEndpoints.push_back(InetAddress("159.203.97.171/9993")); // New York - roots.back().stableEndpoints.push_back(InetAddress("2604:a880:800:a1::54:6001/9993")); // New York - roots.back().stableEndpoints.push_back(InetAddress("169.57.143.104/9993")); // Sao Paolo - roots.back().stableEndpoints.push_back(InetAddress("2607:f0d0:1d01:57::2/9993")); // Sao Paolo - roots.back().stableEndpoints.push_back(InetAddress("107.170.197.14/9993")); // San Francisco - roots.back().stableEndpoints.push_back(InetAddress("2604:a880:1:20::200:e001/9993")); // San Francisco - roots.back().stableEndpoints.push_back(InetAddress("128.199.197.217/9993")); // Singapore - roots.back().stableEndpoints.push_back(InetAddress("2400:6180:0:d0::b7:4001/9993")); // Singapore - - // Bob - roots.push_back(World::Root()); - roots.back().identity = Identity("8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c"); - roots.back().stableEndpoints.push_back(InetAddress("45.32.198.130/9993")); // Dallas - roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:6400:81c3:5400:00ff:fe18:1d61/9993")); // Dallas - roots.back().stableEndpoints.push_back(InetAddress("46.101.160.249/9993")); // Frankfurt - roots.back().stableEndpoints.push_back(InetAddress("2a03:b0c0:3:d0::6a:3001/9993")); // Frankfurt - roots.back().stableEndpoints.push_back(InetAddress("107.191.46.210/9993")); // Paris - roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:6800:83a4::64/9993")); // Paris - roots.back().stableEndpoints.push_back(InetAddress("45.32.246.179/9993")); // Sydney - roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:5800:8bf8:5400:ff:fe15:b39a/9993")); // Sydney - roots.back().stableEndpoints.push_back(InetAddress("45.32.248.87/9993")); // Tokyo - roots.back().stableEndpoints.push_back(InetAddress("2001:19f0:7000:9bc9:5400:00ff:fe15:c4f5/9993")); // Tokyo - roots.back().stableEndpoints.push_back(InetAddress("159.203.2.154/9993")); // Toronto - roots.back().stableEndpoints.push_back(InetAddress("2604:a880:cad:d0::26:7001/9993")); // Toronto - - // END WORLD DEFINITION - // ========================================================================= - - fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts); - - World nw = WorldMaker::make(World::TYPE_PLANET,id,ts,currentKP.pub,roots,previousKP); - - Buffer outtmp; - nw.serialize(outtmp,false); - World testw; - testw.deserialize(outtmp,0); - if (testw != nw) { - fprintf(stderr,"FATAL: serialization test failed!"ZT_EOL_S); - return 1; - } - - OSUtils::writeFile("world.bin",std::string((const char *)outtmp.data(),outtmp.size())); - fprintf(stderr,"INFO: world.bin written with %u bytes of binary world data."ZT_EOL_S,outtmp.size()); - - fprintf(stdout,ZT_EOL_S); - fprintf(stdout,"#define ZT_DEFAULT_WORLD_LENGTH %u"ZT_EOL_S,outtmp.size()); - fprintf(stdout,"static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {"); - for(unsigned int i=0;i 0) - fprintf(stdout,","); - fprintf(stdout,"0x%.2x",(unsigned int)d[i]); - } - fprintf(stdout,"};"ZT_EOL_S); - - return 0; -} diff --git a/world/old/earth-2015-11-16.bin b/world/old/earth-2015-11-16.bin deleted file mode 100644 index 910ff144..00000000 Binary files a/world/old/earth-2015-11-16.bin and /dev/null differ diff --git a/world/old/earth-2015-11-20.bin b/world/old/earth-2015-11-20.bin deleted file mode 100644 index 198682e5..00000000 Binary files a/world/old/earth-2015-11-20.bin and /dev/null differ diff --git a/world/old/earth-2015-12-17.bin b/world/old/earth-2015-12-17.bin deleted file mode 100644 index 20fadb56..00000000 Binary files a/world/old/earth-2015-12-17.bin and /dev/null differ -- cgit v1.2.3 From 471108f2e4b1b9ed56d9d14dfd4531401cea5679 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 30 Jan 2017 08:01:36 -0800 Subject: Slightly increase thread stack size for safety (primary Alpine related) possibly GitHub #443 --- node/Constants.hpp | 5 +++++ osdep/Thread.hpp | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index ac1919b3..a73d4d89 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -419,6 +419,11 @@ #define ZT_UDP_DESIRED_BUF_SIZE 131072 #endif +/** + * Desired / recommended min stack size for threads (used on some platforms to reset thread stack size) + */ +#define ZT_THREAD_MIN_STACK_SIZE 1048576 + /* Ethernet frame types that might be relevant to us */ #define ZT_ETHERTYPE_IPV4 0x0800 #define ZT_ETHERTYPE_ARP 0x0806 diff --git a/osdep/Thread.hpp b/osdep/Thread.hpp index 9f6fb5a8..227c2cfe 100644 --- a/osdep/Thread.hpp +++ b/osdep/Thread.hpp @@ -28,6 +28,7 @@ #include #include #include + #include "../node/Mutex.hpp" namespace ZeroTier { @@ -128,7 +129,7 @@ public: pthread_attr_init(&_tattr); // This corrects for systems with abnormally small defaults (musl) and also // shrinks the stack on systems with large defaults to save a bit of memory. - pthread_attr_setstacksize(&_tattr,524288); + pthread_attr_setstacksize(&_tattr,ZT_THREAD_MIN_STACK_SIZE); _started = false; } -- cgit v1.2.3 From eebd271bb1e3ab0d25f48db07a1a3f1154215bc7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 30 Jan 2017 15:40:22 -0800 Subject: Implement cross cluster sharing of network configs to make clusters able to actually join networks. --- node/Cluster.cpp | 19 +++++++++++++++++++ node/Cluster.hpp | 15 +++++++++++---- node/IncomingPacket.cpp | 9 ++++++++- 3 files changed, 38 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 2a261e51..55503f63 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -44,6 +44,7 @@ #include "Packet.hpp" #include "Switch.hpp" #include "Node.hpp" +#include "Network.hpp" #include "Array.hpp" namespace ZeroTier { @@ -469,6 +470,15 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) RR->sw->send(outp,true); //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); } break; + + case CLUSTER_MESSAGE_NETWORK_CONFIG: { + const SharedPtr network(RR->node->network(dmsg.at(ptr))); + if (network) { + // Copy into a Packet just to conform to Network API. Eventually + // will want to refactor. + network->handleConfigChunk(Packet(dmsg),ptr); + } + } break; } } catch ( ... ) { TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype); @@ -494,6 +504,15 @@ void Cluster::broadcastHavePeer(const Identity &id) } } +void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) +{ + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); + } +} + void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) { if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check diff --git a/node/Cluster.hpp b/node/Cluster.hpp index dafbf425..aba3b8a9 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -216,14 +216,13 @@ public: /** * Replicate a network config for a network we belong to: - * <[8] 64-bit network ID> - * <[2] 16-bit length of network config> - * <[...] serialized network config> + * <[...] network config chunk> * * This is used by clusters to avoid every member having to query * for the same netconf for networks all members belong to. * - * TODO: not implemented yet! + * The first field of a network config chunk is the network ID, + * so this can be checked to look up the network on receipt. */ CLUSTER_MESSAGE_NETWORK_CONFIG = 7 }; @@ -267,6 +266,14 @@ public: */ void broadcastHavePeer(const Identity &id); + /** + * Broadcast a network config chunk to other members of cluster + * + * @param chunk Chunk data + * @param len Length of chunk + */ + void broadcastNetworkConfigChunk(const void *chunk,unsigned int len); + /** * Send this packet via another node in this cluster if another node has this peer * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index a3fbbefc..e703af59 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -461,8 +461,12 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_NETWORK_CONFIG_REQUEST: { const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); - if (network) + if (network) { +#ifdef ZT_ENABLE_CLUSTER + RR->cluster->broadcastNetworkConfigChunk(field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PROTO_VERB_OK_IDX_PAYLOAD),size() - ZT_PROTO_VERB_OK_IDX_PAYLOAD); +#endif network->handleConfigChunk(*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); + } } break; case Packet::VERB_MULTICAST_GATHER: { @@ -922,6 +926,9 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared try { const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); if (network) { +#ifdef ZT_ENABLE_CLUSTER + RR->cluster->broadcastNetworkConfigChunk(field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD),size() - ZT_PACKET_IDX_PAYLOAD); +#endif const uint64_t configUpdateId = network->handleConfigChunk(*this,ZT_PACKET_IDX_PAYLOAD); if (configUpdateId) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); -- cgit v1.2.3 From ed31cb76d6c8e632456a554b79e05df0593b5e9c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 30 Jan 2017 16:04:05 -0800 Subject: Fix to cluster network configs. --- node/Cluster.cpp | 2 +- node/IncomingPacket.cpp | 13 +++---------- node/Network.cpp | 27 +++++++++++++++++++-------- node/Network.hpp | 6 ++++-- 4 files changed, 27 insertions(+), 21 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 55503f63..b9359dc6 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -476,7 +476,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) if (network) { // Copy into a Packet just to conform to Network API. Eventually // will want to refactor. - network->handleConfigChunk(Packet(dmsg),ptr); + network->handleConfigChunk(0,Address(),Packet(dmsg),ptr); } } break; } diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e703af59..c11b0377 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -461,12 +461,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p case Packet::VERB_NETWORK_CONFIG_REQUEST: { const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); - if (network) { -#ifdef ZT_ENABLE_CLUSTER - RR->cluster->broadcastNetworkConfigChunk(field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PROTO_VERB_OK_IDX_PAYLOAD),size() - ZT_PROTO_VERB_OK_IDX_PAYLOAD); -#endif - network->handleConfigChunk(*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); - } + if (network) + network->handleConfigChunk(packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); } break; case Packet::VERB_MULTICAST_GATHER: { @@ -926,10 +922,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared try { const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); if (network) { -#ifdef ZT_ENABLE_CLUSTER - RR->cluster->broadcastNetworkConfigChunk(field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD),size() - ZT_PACKET_IDX_PAYLOAD); -#endif - const uint64_t configUpdateId = network->handleConfigChunk(*this,ZT_PACKET_IDX_PAYLOAD); + const uint64_t configUpdateId = network->handleConfigChunk(packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); if (configUpdateId) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((uint8_t)Packet::VERB_ECHO); diff --git a/node/Network.cpp b/node/Network.cpp index ec1bcb33..320dcf39 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -34,6 +34,7 @@ #include "NetworkController.hpp" #include "Node.hpp" #include "Peer.hpp" +#include "Cluster.hpp" // Uncomment to make the rules engine dump trace info to stdout //#define ZT_RULES_ENGINE_DEBUGGING 1 @@ -908,7 +909,7 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) _myMulticastGroups.erase(i); } -uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) +uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) { const unsigned int start = ptr; @@ -931,12 +932,12 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) chunkIndex = chunk.at(ptr); ptr += 4; if (((chunkIndex + chunkLen) > totalLength)||(totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY)) { // >= since we need room for a null at the end - TRACE("discarded chunk from %s: invalid length or length overflow",chunk.source().toString().c_str()); + TRACE("discarded chunk from %s: invalid length or length overflow",source.toString().c_str()); return 0; } if ((chunk[ptr] != 1)||(chunk.at(ptr + 1) != ZT_C25519_SIGNATURE_LEN)) { - TRACE("discarded chunk from %s: unrecognized signature type",chunk.source().toString().c_str()); + TRACE("discarded chunk from %s: unrecognized signature type",source.toString().c_str()); return 0; } const uint8_t *sig = reinterpret_cast(chunk.field(ptr + 3,ZT_C25519_SIGNATURE_LEN)); @@ -964,30 +965,35 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) // If it's not a duplicate, check chunk signature const Identity controllerId(RR->topology->getIdentity(controller())); if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? - TRACE("unable to verify chunk from %s: don't have controller identity",chunk.source().toString().c_str()); + TRACE("unable to verify chunk from %s: don't have controller identity",source.toString().c_str()); return 0; } if (!controllerId.verify(chunk.field(start,ptr - start),ptr - start,sig,ZT_C25519_SIGNATURE_LEN)) { - TRACE("discarded chunk from %s: signature check failed",chunk.source().toString().c_str()); + TRACE("discarded chunk from %s: signature check failed",source.toString().c_str()); return 0; } +#ifdef ZT_ENABLE_CLUSTER + if (source) + RR->cluster->broadcastNetworkConfigChunk(chunk.field(start,chunk.size() - start),chunk.size() - start); +#endif + // New properly verified chunks can be flooded "virally" through the network if (fastPropagate) { Address *a = (Address *)0; Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - if ((*a != chunk.source())&&(*a != controller())) { + if ((*a != source)&&(*a != controller())) { Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CONFIG); outp.append(reinterpret_cast(chunk.data()) + start,chunk.size() - start); RR->sw->send(outp,true); } } } - } else if (chunk.source() == controller()) { + } else if ((source == controller())||(!source)) { // since old chunks aren't signed, only accept from controller itself (or via cluster backplane) // Legacy support for OK(NETWORK_CONFIG_REQUEST) from older controllers - chunkId = chunk.packetId(); + chunkId = packetId; configUpdateId = chunkId; totalLength = chunkLen; chunkIndex = 0; @@ -999,6 +1005,11 @@ uint64_t Network::handleConfigChunk(const Packet &chunk,unsigned int ptr) if ((!c)||(_incomingConfigChunks[i].ts < c->ts)) c = &(_incomingConfigChunks[i]); } + +#ifdef ZT_ENABLE_CLUSTER + if (source) + RR->cluster->broadcastNetworkConfigChunk(chunk.field(start,chunk.size() - start),chunk.size() - start); +#endif } else { TRACE("discarded single-chunk unsigned legacy config: this is only allowed if the sender is the controller itself"); return 0; diff --git a/node/Network.hpp b/node/Network.hpp index 1627be58..85ee6e9a 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -181,11 +181,13 @@ public: * chunks via OK(NETWORK_CONFIG_REQUEST) or NETWORK_CONFIG. It verifies * each chunk and once assembled applies the configuration. * - * @param chunk Packet containing chunk + * @param packetId Packet ID or 0 if none (e.g. via cluster path) + * @param source Address of sender of chunk or NULL if none (e.g. via cluster path) + * @param chunk Buffer containing chunk * @param ptr Index of chunk and related fields in packet * @return Update ID if update was fully assembled and accepted or 0 otherwise */ - uint64_t handleConfigChunk(const Packet &chunk,unsigned int ptr); + uint64_t handleConfigChunk(const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr); /** * Set network configuration -- cgit v1.2.3 From f9ad80aa13705102c060bf6b16e62bb2bc3edfff Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 30 Jan 2017 16:15:47 -0800 Subject: . --- node/Cluster.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index b9359dc6..5537782e 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -476,7 +476,8 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) if (network) { // Copy into a Packet just to conform to Network API. Eventually // will want to refactor. - network->handleConfigChunk(0,Address(),Packet(dmsg),ptr); + printf("<< CLUSTER_MESSAGE_NETWORK_CONFIG %.16llx\n",dmsg.at(ptr)); + network->handleConfigChunk(0,Address(),Buffer(dmsg),ptr); } } break; } @@ -511,6 +512,7 @@ void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) Mutex::Lock _l2(_members[*mid].lock); _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); } + printf(">> CLUSTER_MESSAGE_NETWORK_CONFIG\n"); } void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) -- cgit v1.2.3 From 6d5a3cd2e2fe196325cb3ab60dfc368901395977 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 30 Jan 2017 16:23:38 -0800 Subject: Remove debug code. Cluster network config sharing seems to work. --- node/Cluster.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 5537782e..56a6a06d 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -476,7 +476,6 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) if (network) { // Copy into a Packet just to conform to Network API. Eventually // will want to refactor. - printf("<< CLUSTER_MESSAGE_NETWORK_CONFIG %.16llx\n",dmsg.at(ptr)); network->handleConfigChunk(0,Address(),Buffer(dmsg),ptr); } } break; @@ -512,7 +511,6 @@ void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) Mutex::Lock _l2(_members[*mid].lock); _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); } - printf(">> CLUSTER_MESSAGE_NETWORK_CONFIG\n"); } void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) -- cgit v1.2.3 From 5dbebc513ad67bb65c41c42a9c761c085309564f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 12:00:25 -0800 Subject: Minor send path refactor to make packet I/O work on clusters if they are members of networks. Also fix a crash if compiled in cluster mode but no cluster is enabled. --- node/Cluster.cpp | 201 +++++++++++++++++++++++++++++++++++++++---------------- node/Cluster.hpp | 42 +++++++++++- node/Network.cpp | 4 +- node/Switch.cpp | 126 ++++++++++++++++++++++------------ node/Switch.hpp | 6 +- 5 files changed, 271 insertions(+), 108 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 56a6a06d..16ef040b 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -342,17 +342,20 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) Identity id; ptr += id.deserialize(dmsg,ptr); if (id) { - RR->topology->saveIdentity(id); - { Mutex::Lock _l(_remotePeers_m); - _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)] = RR->node->now(); + _RemotePeer &rp = _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)]; + if (!rp.lastHavePeerReceived) { + RR->topology->saveIdentity(id); + id.agree(RR->identity,rp.key,ZT_PEER_SECRET_KEY_LENGTH); + } + rp.lastHavePeerReceived = RR->node->now(); } _ClusterSendQueueEntry *q[16384]; // 16384 is "tons" unsigned int qc = _sendQueue->getByDest(id.address(),q,16384); for(unsigned int i=0;isendViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite); + this->relayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite); _sendQueue->returnToPool(q,qc); TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc); @@ -513,7 +516,79 @@ void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) } } -void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) +int Cluster::prepSendViaCluster(const Address &toPeerAddress,Packet &outp,bool encrypt) +{ + const uint64_t now = RR->node->now(); + uint64_t mostRecentTs = 0; + int mostRecentMemberId = -1; + uint8_t mostRecentSecretKey[ZT_PEER_SECRET_KEY_LENGTH]; + { + Mutex::Lock _l2(_remotePeers_m); + std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); + for(;;) { + if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) + break; + else if (rpe->second.lastHavePeerReceived > mostRecentTs) { + mostRecentTs = rpe->second.lastHavePeerReceived; + memcpy(mostRecentSecretKey,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); + mostRecentMemberId = (int)rpe->first.second; + } + ++rpe; + } + } + + if (mostRecentMemberId >= 0) { + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) + return -1; + + bool sendWantPeer = true; + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER + } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + } + } + } + } + + outp.armor(mostRecentSecretKey,encrypt); + return mostRecentMemberId; + } else return -1; +} + +bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len) +{ + if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check + return false; + Mutex::Lock _l2(_members[mostRecentMemberId].lock); + for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { + for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { + if (i1->ss_family == i2->ss_family) { + TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); + RR->node->putPacket(*i1,*i2,data,len); + return true; + } + } + } + return false; +} + +void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) { if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check return; @@ -521,87 +596,101 @@ void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee const uint64_t now = RR->node->now(); uint64_t mostRecentTs = 0; - unsigned int mostRecentMemberId = 0xffffffff; + int mostRecentMemberId = -1; { Mutex::Lock _l2(_remotePeers_m); - std::map< std::pair,uint64_t >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); + std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); for(;;) { if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) break; - else if (rpe->second > mostRecentTs) { - mostRecentTs = rpe->second; - mostRecentMemberId = rpe->first.second; + else if (rpe->second.lastHavePeerReceived > mostRecentTs) { + mostRecentTs = rpe->second.lastHavePeerReceived; + mostRecentMemberId = (int)rpe->first.second; } ++rpe; } } - const uint64_t age = now - mostRecentTs; - if (age >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { - const bool enqueueAndWait = ((age >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId > 0xffff)); + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + // Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing + const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0)); // Poll everyone with WANT_PEER if the age of our most recent entry is // approaching expiration (or has expired, or does not exist). - char tmp[ZT_ADDRESS_LENGTH]; - toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + bool sendWantPeer = true; { - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER + } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + } } } // If there isn't a good place to send via, then enqueue this for retrying // later and return after having broadcasted a WANT_PEER. if (enqueueAndWait) { - TRACE("sendViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str()); + TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str()); _sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite); return; } } - Buffer<1024> buf; - if (unite) { - InetAddress v4,v6; - if (fromPeerAddress) { - SharedPtr fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); - if (fromPeer) - fromPeer->getBestActiveAddresses(now,v4,v6); - } - uint8_t addrCount = 0; - if (v4) - ++addrCount; - if (v6) - ++addrCount; - if (addrCount) { - toPeerAddress.appendTo(buf); - fromPeerAddress.appendTo(buf); - buf.append(addrCount); + if (mostRecentMemberId >= 0) { + Buffer<1024> buf; + if (unite) { + InetAddress v4,v6; + if (fromPeerAddress) { + SharedPtr fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); + if (fromPeer) + fromPeer->getBestActiveAddresses(now,v4,v6); + } + uint8_t addrCount = 0; if (v4) - v4.serialize(buf); + ++addrCount; if (v6) - v6.serialize(buf); + ++addrCount; + if (addrCount) { + toPeerAddress.appendTo(buf); + fromPeerAddress.appendTo(buf); + buf.append(addrCount); + if (v4) + v4.serialize(buf); + if (v6) + v6.serialize(buf); + } } - } - { - Mutex::Lock _l2(_members[mostRecentMemberId].lock); - if (buf.size() > 0) - _send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size()); - - for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { - for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { - if (i1->ss_family == i2->ss_family) { - TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket(*i1,*i2,data,len); - return; + { + Mutex::Lock _l2(_members[mostRecentMemberId].lock); + if (buf.size() > 0) + _send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size()); + + for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { + for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { + if (i1->ss_family == i2->ss_family) { + TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); + RR->node->putPacket(*i1,*i2,data,len); + return; + } } } - } - TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId); - return; + TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId); + } } } @@ -663,8 +752,8 @@ void Cluster::doPeriodicTasks() _lastCleanedRemotePeers = now; Mutex::Lock _l(_remotePeers_m); - for(std::map< std::pair,uint64_t >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) { - if ((now - rp->second) >= ZT_PEER_ACTIVITY_TIMEOUT) + for(std::map< std::pair,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) { + if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT) _remotePeers.erase(rp++); else ++rp; } diff --git a/node/Cluster.hpp b/node/Cluster.hpp index aba3b8a9..7ebef0c9 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -88,6 +88,11 @@ */ #define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500 +/** + * We won't send WANT_PEER to other members more than every (ms) per recipient + */ +#define ZT_CLUSTER_WANT_PEER_EVERY 1000 + namespace ZeroTier { class RuntimeEnvironment; @@ -275,7 +280,30 @@ public: void broadcastNetworkConfigChunk(const void *chunk,unsigned int len); /** - * Send this packet via another node in this cluster if another node has this peer + * If the cluster has this peer, prepare the packet to send via cluster + * + * Note that outp is only armored (or modified at all) if the return value is a member ID. + * + * @param toPeerAddress Value of outp.destination(), simply to save additional lookup + * @param outp Packet to armor with peer key (via cluster knowledge of peer shared secret) + * @param encrypt If true, encrypt packet payload (passed to Packet::armor()) + * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() + */ + int prepSendViaCluster(const Address &toPeerAddress,Packet &outp,bool encrypt); + + /** + * Send data via cluster front plane (packet head or fragment) + * + * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster() + * @param toPeerAddress Destination peer address + * @param data Packet or packet fragment data + * @param len Length of packet or fragment + * @return True if packet was sent (and outp was modified via armoring) + */ + bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len); + + /** + * Relay a packet via the cluster * * This is used in the outgoing packet and relaying logic in Switch to * relay packets to other cluster members. It isn't PROXY_SEND-- that is @@ -287,7 +315,7 @@ public: * @param len Length of packet or fragment * @param unite If true, also request proxy unite across cluster */ - void sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite); + void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite); /** * Send a distributed query to other cluster members @@ -398,7 +426,15 @@ private: std::vector _memberIds; Mutex _memberIds_m; - std::map< std::pair,uint64_t > _remotePeers; // we need ordered behavior and lower_bound here + struct _RemotePeer + { + _RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {} + ~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); } + uint64_t lastHavePeerReceived; + uint64_t lastSentWantPeer; + uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement + }; + std::map< std::pair,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here Mutex _remotePeers_m; uint64_t _lastFlushed; diff --git a/node/Network.cpp b/node/Network.cpp index 320dcf39..c5855418 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -974,7 +974,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc } #ifdef ZT_ENABLE_CLUSTER - if (source) + if ((source)&&(RR->cluster)) RR->cluster->broadcastNetworkConfigChunk(chunk.field(start,chunk.size() - start),chunk.size() - start); #endif @@ -1007,7 +1007,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc } #ifdef ZT_ENABLE_CLUSTER - if (source) + if ((source)&&(RR->cluster)) RR->cluster->broadcastNetworkConfigChunk(chunk.field(start,chunk.size() - start),chunk.size() - start); #endif } else { diff --git a/node/Switch.cpp b/node/Switch.cpp index f935b7aa..6df84101 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -117,7 +117,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from if ((!relayTo)||(!relayTo->sendDirect(fragment.data(),fragment.size(),now,false))) { #ifdef ZT_ENABLE_CLUSTER if (RR->cluster) { - RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false); + RR->cluster->relayViaCluster(Address(),destination,fragment.data(),fragment.size(),false); return; } #endif @@ -204,7 +204,6 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); - if (destination != RR->identity.address()) { if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) return; @@ -233,7 +232,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from if (shouldUnite) luts = now; } - RR->cluster->sendViaCluster(source,destination,packet.data(),packet.size(),shouldUnite); + RR->cluster->relayViaCluster(source,destination,packet.data(),packet.size(),shouldUnite); return; } #endif @@ -560,7 +559,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } } -void Switch::send(const Packet &packet,bool encrypt) +void Switch::send(Packet &packet,bool encrypt) { if (packet.destination() == RR->identity.address()) { TRACE("BUG: caught attempt to send() to self, ignored"); @@ -687,12 +686,17 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread return Address(); } -bool Switch::_trySend(const Packet &packet,bool encrypt) +bool Switch::_trySend(Packet &packet,bool encrypt) { - const SharedPtr peer(RR->topology->getPeer(packet.destination())); - if (peer) { - const uint64_t now = RR->node->now(); + SharedPtr viaPath; + const uint64_t now = RR->node->now(); + const Address destination(packet.destination()); +#ifdef ZT_ENABLE_CLUSTER + int clusterMostRecentMemberId = -1; +#endif + const SharedPtr peer(RR->topology->getPeer(destination)); + if (peer) { /* First get the best path, and if it's dead (and this is not a root) * we attempt to re-activate that path but this packet will flow * upstream. If the path comes back alive, it will be used in the future. @@ -700,58 +704,92 @@ bool Switch::_trySend(const Packet &packet,bool encrypt) * to send heartbeats "down" and because we have to at least try to * go somewhere. */ - SharedPtr viaPath(peer->getBestPath(now,false)); + viaPath = peer->getBestPath(now,false); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now); viaPath.zero(); } + if (!viaPath) { - peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known - const SharedPtr relay(RR->topology->getUpstreamPeer()); - if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { - if (!(viaPath = peer->getBestPath(now,true))) - return false; +#ifdef ZT_ENABLE_CLUSTER + if (RR->cluster) + clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,packet,encrypt); + if (clusterMostRecentMemberId < 0) { +#endif + peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known + const SharedPtr relay(RR->topology->getUpstreamPeer()); + if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { + if (!(viaPath = peer->getBestPath(now,true))) // last resort: try an expired path... we usually can never get here + return false; + } +#ifdef ZT_ENABLE_CLUSTER } +#endif } + } else { + requestWhois(destination); +#ifndef ZT_ENABLE_CLUSTER + return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly +#endif + } - Packet tmp(packet); +#ifdef ZT_TRACE +#ifdef ZT_ENABLE_CLUSTER + if ((!viaPath)&&(clusterMostRecentMemberId < 0)) { + TRACE("BUG: both viaPath and clusterMostRecentMemberId ended up invalid in Switch::_trySend()!"); + abort(); + } +#else + if (!viaPath) { + TRACE("BUG: viaPath ended up NULL in Switch::_trySend()!"); + abort(); + } +#endif +#endif - unsigned int chunkSize = std::min(tmp.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); - tmp.setFragmented(chunkSize < tmp.size()); + unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); + packet.setFragmented(chunkSize < packet.size()); - const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address()); - if (trustedPathId) { - tmp.setTrusted(trustedPathId); - } else { - tmp.armor(peer->key(),encrypt); - } + const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address()); + if (trustedPathId) { + packet.setTrusted(trustedPathId); + } else { + packet.armor(peer->key(),encrypt); + } - if (viaPath->send(RR,tmp.data(),chunkSize,now)) { - if (chunkSize < tmp.size()) { - // Too big for one packet, fragment the rest - unsigned int fragStart = chunkSize; - unsigned int remaining = tmp.size() - chunkSize; - unsigned int fragsRemaining = (remaining / (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)); - if ((fragsRemaining * (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining) - ++fragsRemaining; - const unsigned int totalFragments = fragsRemaining + 1; - - for(unsigned int fno=1;fnosend(RR,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { +#else + if (viaPath->send(RR,packet.data(),chunkSize,now)) { +#endif + if (chunkSize < packet.size()) { + // Too big for one packet, fragment the rest + unsigned int fragStart = chunkSize; + unsigned int remaining = packet.size() - chunkSize; + unsigned int fragsRemaining = (remaining / (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)); + if ((fragsRemaining * (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining) + ++fragsRemaining; + const unsigned int totalFragments = fragsRemaining + 1; + + for(unsigned int fno=1;fnosend(RR,frag.data(),frag.size(),now); - fragStart += chunkSize; - remaining -= chunkSize; - } + else if (clusterMostRecentMemberId >= 0) + RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,frag.data(),frag.size()); +#else + viaPath->send(RR,frag.data(),frag.size(),now); +#endif + fragStart += chunkSize; + remaining -= chunkSize; } - - return true; } - } else { - requestWhois(packet.destination()); } - return false; + + return true; } bool Switch::_unite(const Address &p1,const Address &p2) diff --git a/node/Switch.hpp b/node/Switch.hpp index f44eef48..422f6c8e 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -92,10 +92,10 @@ public: * Needless to say, the packet's source must be this node. Otherwise it * won't be encrypted right. (This is not used for relaying.) * - * @param packet Packet to send + * @param packet Packet to send (buffer may be modified) * @param encrypt Encrypt packet payload? (always true except for HELLO) */ - void send(const Packet &packet,bool encrypt); + void send(Packet &packet,bool encrypt); /** * Request WHOIS on a given address @@ -126,7 +126,7 @@ public: private: Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); - bool _trySend(const Packet &packet,bool encrypt); + bool _trySend(Packet &packet,bool encrypt); // packet is modified if return is true bool _unite(const Address &p1,const Address &p2); const RuntimeEnvironment *const RR; -- cgit v1.2.3 From 5e11cf637816121f79c3ed00370843e93b62b1c6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 12:32:06 -0800 Subject: Can't armor() a packet until all flags are set. --- node/Cluster.cpp | 6 ++---- node/Cluster.hpp | 5 ++--- node/Switch.cpp | 12 +++++++++++- 3 files changed, 15 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 16ef040b..a24cf99d 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -516,12 +516,11 @@ void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) } } -int Cluster::prepSendViaCluster(const Address &toPeerAddress,Packet &outp,bool encrypt) +int Cluster::prepSendViaCluster(const Address &toPeerAddress,void *peerSecret) { const uint64_t now = RR->node->now(); uint64_t mostRecentTs = 0; int mostRecentMemberId = -1; - uint8_t mostRecentSecretKey[ZT_PEER_SECRET_KEY_LENGTH]; { Mutex::Lock _l2(_remotePeers_m); std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); @@ -530,7 +529,7 @@ int Cluster::prepSendViaCluster(const Address &toPeerAddress,Packet &outp,bool e break; else if (rpe->second.lastHavePeerReceived > mostRecentTs) { mostRecentTs = rpe->second.lastHavePeerReceived; - memcpy(mostRecentSecretKey,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); + memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); mostRecentMemberId = (int)rpe->first.second; } ++rpe; @@ -566,7 +565,6 @@ int Cluster::prepSendViaCluster(const Address &toPeerAddress,Packet &outp,bool e } } - outp.armor(mostRecentSecretKey,encrypt); return mostRecentMemberId; } else return -1; } diff --git a/node/Cluster.hpp b/node/Cluster.hpp index 7ebef0c9..90ea3e7d 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -285,11 +285,10 @@ public: * Note that outp is only armored (or modified at all) if the return value is a member ID. * * @param toPeerAddress Value of outp.destination(), simply to save additional lookup - * @param outp Packet to armor with peer key (via cluster knowledge of peer shared secret) - * @param encrypt If true, encrypt packet payload (passed to Packet::armor()) + * @param peerSecret Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() */ - int prepSendViaCluster(const Address &toPeerAddress,Packet &outp,bool encrypt); + int prepSendViaCluster(const Address &toPeerAddress,void *peerSecret); /** * Send data via cluster front plane (packet head or fragment) diff --git a/node/Switch.cpp b/node/Switch.cpp index 6df84101..d4f477f0 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -693,6 +693,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) const Address destination(packet.destination()); #ifdef ZT_ENABLE_CLUSTER int clusterMostRecentMemberId = -1; + uint8_t clusterPeerSecret[ZT_PEER_SECRET_KEY_LENGTH]; #endif const SharedPtr peer(RR->topology->getPeer(destination)); @@ -714,7 +715,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) if (!viaPath) { #ifdef ZT_ENABLE_CLUSTER if (RR->cluster) - clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,packet,encrypt); + clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterPeerSecret); if (clusterMostRecentMemberId < 0) { #endif peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known @@ -751,12 +752,21 @@ bool Switch::_trySend(Packet &packet,bool encrypt) unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); packet.setFragmented(chunkSize < packet.size()); +#ifdef ZT_ENABLE_CLUSTER + const uint64_t trustedPathId = (viaPath) ? RR->topology->getOutboundPathTrust(viaPath->address()) : 0; + if (trustedPathId) { + packet.setTrusted(trustedPathId); + } else { + packet.armor((clusterMostRecentMemberId >= 0) ? clusterPeerSecret : peer->key(),encrypt); + } +#else const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address()); if (trustedPathId) { packet.setTrusted(trustedPathId); } else { packet.armor(peer->key(),encrypt); } +#endif #ifdef ZT_ENABLE_CLUSTER if ( ((viaPath)&&(viaPath->send(RR,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { -- cgit v1.2.3 From e778d45128295ddf9bbb00f4a46f103de4bc4c11 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 12:51:52 -0800 Subject: Still want to send WANT_PEER under two failure modes. --- node/Cluster.cpp | 48 +++++++++++++++++++++++------------------------- node/Switch.cpp | 27 ++++++++++++++------------- 2 files changed, 37 insertions(+), 38 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index a24cf99d..ef09e842 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -536,37 +536,35 @@ int Cluster::prepSendViaCluster(const Address &toPeerAddress,void *peerSecret) } } - if (mostRecentMemberId >= 0) { - const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; - if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { - if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) - return -1; + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) + mostRecentMemberId = -1; - bool sendWantPeer = true; - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; - if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { - rp.lastSentWantPeer = now; - } else { - sendWantPeer = false; // don't flood WANT_PEER - } + bool sendWantPeer = true; + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER } - if (sendWantPeer) { - char tmp[ZT_ADDRESS_LENGTH]; - toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); - { - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); - } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); } } } + } - return mostRecentMemberId; - } else return -1; + return mostRecentMemberId; } bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len) diff --git a/node/Switch.cpp b/node/Switch.cpp index d4f477f0..6185037d 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -729,24 +729,25 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #endif } } else { - requestWhois(destination); -#ifndef ZT_ENABLE_CLUSTER - return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly +#ifdef ZT_ENABLE_CLUSTER + if (RR->cluster) + clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterPeerSecret); + if (clusterMostRecentMemberId < 0) { +#else + requestWhois(destination); + return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly +#endif +#ifdef ZT_ENABLE_CLUSTER + } #endif } -#ifdef ZT_TRACE #ifdef ZT_ENABLE_CLUSTER - if ((!viaPath)&&(clusterMostRecentMemberId < 0)) { - TRACE("BUG: both viaPath and clusterMostRecentMemberId ended up invalid in Switch::_trySend()!"); - abort(); - } + if ((!viaPath)&&(clusterMostRecentMemberId < 0)) + return false; #else - if (!viaPath) { - TRACE("BUG: viaPath ended up NULL in Switch::_trySend()!"); - abort(); - } -#endif + if (!viaPath) + return false; #endif unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); -- cgit v1.2.3 From b378f5dcd761b5ad76469708c98bf923ec1d7896 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 13:20:51 -0800 Subject: Take 3 --- node/Cluster.cpp | 4 ++-- node/Cluster.hpp | 5 +++-- node/Switch.cpp | 36 ++++++++++++++++++++++-------------- 3 files changed, 27 insertions(+), 18 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index ef09e842..75a9f29b 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -516,10 +516,10 @@ void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) } } -int Cluster::prepSendViaCluster(const Address &toPeerAddress,void *peerSecret) +int Cluster::prepSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) { const uint64_t now = RR->node->now(); - uint64_t mostRecentTs = 0; + mostRecentTs = 0; int mostRecentMemberId = -1; { Mutex::Lock _l2(_remotePeers_m); diff --git a/node/Cluster.hpp b/node/Cluster.hpp index 90ea3e7d..72fcd04d 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -285,10 +285,11 @@ public: * Note that outp is only armored (or modified at all) if the return value is a member ID. * * @param toPeerAddress Value of outp.destination(), simply to save additional lookup - * @param peerSecret Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes + * @param ts Result: set to time of last HAVE_PEER from the cluster + * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() */ - int prepSendViaCluster(const Address &toPeerAddress,void *peerSecret); + int prepSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); /** * Send data via cluster front plane (packet head or fragment) diff --git a/node/Switch.cpp b/node/Switch.cpp index 6185037d..1d9f6436 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -692,6 +692,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) const uint64_t now = RR->node->now(); const Address destination(packet.destination()); #ifdef ZT_ENABLE_CLUSTER + uint64_t clusterMostRecentTs = 0; int clusterMostRecentMemberId = -1; uint8_t clusterPeerSecret[ZT_PEER_SECRET_KEY_LENGTH]; #endif @@ -713,25 +714,31 @@ bool Switch::_trySend(Packet &packet,bool encrypt) } if (!viaPath) { + const SharedPtr relay(RR->topology->getUpstreamPeer()); + if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) + viaPath = peer->getBestPath(now,true); + } + #ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterPeerSecret); - if (clusterMostRecentMemberId < 0) { -#endif - peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known - const SharedPtr relay(RR->topology->getUpstreamPeer()); - if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { - if (!(viaPath = peer->getBestPath(now,true))) // last resort: try an expired path... we usually can never get here - return false; - } -#ifdef ZT_ENABLE_CLUSTER - } -#endif + if (RR->cluster) + clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); + if (clusterMostRecentMemberId >= 0) { + if ((viaPath)&&(viaPath->lastIn() < clusterMostRecentTs)) + viaPath.zero(); + } else if (!viaPath) { + peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known + return false; } +#else + if (!viaPath) { + peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known + return false; + } +#endif } else { #ifdef ZT_ENABLE_CLUSTER if (RR->cluster) - clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterPeerSecret); + clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); if (clusterMostRecentMemberId < 0) { #else requestWhois(destination); @@ -742,6 +749,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #endif } + // Sanity checks #ifdef ZT_ENABLE_CLUSTER if ((!viaPath)&&(clusterMostRecentMemberId < 0)) return false; -- cgit v1.2.3 From 60ff280dcb213a3e50eb410184493ace1d4c2736 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 13:52:53 -0800 Subject: Another tweak to cluster I/O rules. --- node/Cluster.cpp | 2 +- node/Cluster.hpp | 2 +- node/Peer.hpp | 2 +- node/Switch.cpp | 47 ++++++++++++++++++++++------------------------- 4 files changed, 25 insertions(+), 28 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 75a9f29b..3e33d802 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -516,7 +516,7 @@ void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) } } -int Cluster::prepSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) +int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) { const uint64_t now = RR->node->now(); mostRecentTs = 0; diff --git a/node/Cluster.hpp b/node/Cluster.hpp index 72fcd04d..cd6a3bb9 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -289,7 +289,7 @@ public: * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() */ - int prepSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); + int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); /** * Send data via cluster front plane (packet head or fragment) diff --git a/node/Peer.hpp b/node/Peer.hpp index 78b345b9..06abde3f 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -137,7 +137,7 @@ public: * Get the best current direct path * * @param now Current time - * @param includeDead If true, include even expired paths + * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none */ SharedPtr getBestPath(uint64_t now,bool includeExpired); diff --git a/node/Switch.cpp b/node/Switch.cpp index 1d9f6436..6860b865 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -691,10 +691,13 @@ bool Switch::_trySend(Packet &packet,bool encrypt) SharedPtr viaPath; const uint64_t now = RR->node->now(); const Address destination(packet.destination()); + #ifdef ZT_ENABLE_CLUSTER uint64_t clusterMostRecentTs = 0; int clusterMostRecentMemberId = -1; uint8_t clusterPeerSecret[ZT_PEER_SECRET_KEY_LENGTH]; + if (RR->cluster) + clusterMostRecentMemberId = RR->cluster->checkSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); #endif const SharedPtr peer(RR->topology->getPeer(destination)); @@ -708,37 +711,40 @@ bool Switch::_trySend(Packet &packet,bool encrypt) viaPath = peer->getBestPath(now,false); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { - if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) - peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now); +#ifdef ZT_ENABLE_CLUSTER + if ((clusterMostRecentMemberId < 0)||(viaPath->lastIn() > clusterMostRecentTs)) { +#endif + if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { + peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now); + viaPath->sent(now); + } +#ifdef ZT_ENABLE_CLUSTER + } +#endif viaPath.zero(); } - if (!viaPath) { - const SharedPtr relay(RR->topology->getUpstreamPeer()); - if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) - viaPath = peer->getBestPath(now,true); - } - #ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); if (clusterMostRecentMemberId >= 0) { if ((viaPath)&&(viaPath->lastIn() < clusterMostRecentTs)) viaPath.zero(); } else if (!viaPath) { - peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known - return false; - } #else if (!viaPath) { +#endif peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known - return false; + const SharedPtr relay(RR->topology->getUpstreamPeer()); + if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { + if (!(viaPath = peer->getBestPath(now,true))) + return false; + } +#ifdef ZT_ENABLE_CLUSTER + } +#else } #endif } else { #ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - clusterMostRecentMemberId = RR->cluster->prepSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); if (clusterMostRecentMemberId < 0) { #else requestWhois(destination); @@ -749,15 +755,6 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #endif } - // Sanity checks -#ifdef ZT_ENABLE_CLUSTER - if ((!viaPath)&&(clusterMostRecentMemberId < 0)) - return false; -#else - if (!viaPath) - return false; -#endif - unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); packet.setFragmented(chunkSize < packet.size()); -- cgit v1.2.3 From fc3f4fb988c77d35792e593c17715a223a126d60 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 14:05:13 -0800 Subject: Yeah that could never have worked (normal packets in cluster mode). --- node/Switch.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 6860b865..fdf889ea 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -197,11 +197,6 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from const Address destination(reinterpret_cast(data) + 8,ZT_ADDRESS_LENGTH); const Address source(reinterpret_cast(data) + 13,ZT_ADDRESS_LENGTH); - // Catch this and toss it -- it would never work, but it could happen if we somehow - // mistakenly guessed an address we're bound to as a destination for another peer. - if (source == RR->identity.address()) - return; - //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); if (destination != RR->identity.address()) { @@ -223,7 +218,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } } else { #ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) { + if ((RR->cluster)&&(source != RR->identity.address())) { bool shouldUnite; { Mutex::Lock _l(_lastUniteAttempt_m); -- cgit v1.2.3 From 29ec7bf3a2e70fcf3b38a39cf00e8dd7ac9e0818 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 14:18:56 -0800 Subject: Add more specific check in source==self case instead of dumping it. --- node/Cluster.cpp | 13 +++++++++++++ node/Cluster.hpp | 6 ++++++ node/Switch.cpp | 13 +++++++++++++ 3 files changed, 32 insertions(+) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 3e33d802..7cf6b08b 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -862,6 +862,19 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr } } +bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const +{ + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + for(std::vector::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) { + if (ip == *i2) + return true; + } + } + return false; +} + void Cluster::status(ZT_ClusterStatus &status) const { const uint64_t now = RR->node->now(); diff --git a/node/Cluster.hpp b/node/Cluster.hpp index cd6a3bb9..08e32a99 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -358,6 +358,12 @@ public: */ bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload); + /** + * @param ip Address to check + * @return True if this is a cluster frontplane address (excluding our addresses) + */ + bool isClusterPeerFrontplane(const InetAddress &ip) const; + /** * Fill out ZT_ClusterStatus structure (from core API) * diff --git a/node/Switch.cpp b/node/Switch.cpp index fdf889ea..cddf0f66 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -199,6 +199,14 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); +#ifdef ZT_ENABLE_CLUSTER + if ( (source == RR->identity.address()) && ((!RR->cluster)||(!RR->cluster->isClusterPeerFrontplane(fromAddr))) ) + return; +#else + if (source == RR->identity.address()) + return; +#endif + if (destination != RR->identity.address()) { if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) return; @@ -206,7 +214,12 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from Packet packet(data,len); if (packet.hops() < ZT_RELAY_MAX_HOPS) { +#ifdef ZT_ENABLE_CLUSTER + if (source != RR->identity.address()) + packet.incrementHops(); +#else packet.incrementHops(); +#endif SharedPtr relayTo = RR->topology->getPeer(destination); if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { -- cgit v1.2.3 From 62a705af1c88dd3b6cd40d1d3b1ce5e602a5ad31 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 14:35:07 -0800 Subject: Eliminate another check in cluster frontplane mode. --- node/Switch.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index cddf0f66..56a15f0a 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -105,7 +105,13 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from const Address destination(fragment.destination()); if (destination != RR->identity.address()) { - if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) +#ifdef ZT_ENABLE_CLUSTER + const bool isClusterFrontplane = ((RR->cluster)&&(RR->cluster->isClusterPeerFrontplane(fromAddr))); +#else + const bool isClusterFrontplane = false; +#endif + + if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) && (!isClusterFrontplane) ) return; if (fragment.hops() < ZT_RELAY_MAX_HOPS) { @@ -116,7 +122,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from SharedPtr relayTo = RR->topology->getPeer(destination); if ((!relayTo)||(!relayTo->sendDirect(fragment.data(),fragment.size(),now,false))) { #ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) { + if ((RR->cluster)&&(!isClusterFrontplane)) { RR->cluster->relayViaCluster(Address(),destination,fragment.data(),fragment.size(),false); return; } @@ -208,7 +214,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif if (destination != RR->identity.address()) { - if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) + if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) && (source != RR->identity.address()) ) return; Packet packet(data,len); @@ -223,11 +229,13 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from SharedPtr relayTo = RR->topology->getPeer(destination); if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { - Mutex::Lock _l(_lastUniteAttempt_m); - uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; - if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) { - luts = now; - _unite(source,destination); + if (source != RR->identity.address()) { + Mutex::Lock _l(_lastUniteAttempt_m); + uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; + if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) { + luts = now; + _unite(source,destination); + } } } else { #ifdef ZT_ENABLE_CLUSTER -- cgit v1.2.3 From 9284e4edfe8267f4f33ac563a17571fcf73833a0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Feb 2017 15:22:14 -0800 Subject: agree() must be called on our identity, the one with the secret --- node/Cluster.cpp | 2 +- node/Switch.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 7cf6b08b..356a0887 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -347,7 +347,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) _RemotePeer &rp = _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)]; if (!rp.lastHavePeerReceived) { RR->topology->saveIdentity(id); - id.agree(RR->identity,rp.key,ZT_PEER_SECRET_KEY_LENGTH); + RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH); } rp.lastHavePeerReceived = RR->node->now(); } diff --git a/node/Switch.cpp b/node/Switch.cpp index 56a15f0a..2ab3ccfc 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -82,12 +82,12 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from * no longer send these, but we'll listen for them for a while to * locate peers with versions <1.0.4. */ - Address beaconAddr(reinterpret_cast(data) + 8,5); + const Address beaconAddr(reinterpret_cast(data) + 8,5); if (beaconAddr == RR->identity.address()) return; if (!RR->node->shouldUsePathForZeroTierTraffic(beaconAddr,localAddr,fromAddr)) return; - SharedPtr peer(RR->topology->getPeer(beaconAddr)); + const SharedPtr peer(RR->topology->getPeer(beaconAddr)); if (peer) { // we'll only respond to beacons from known peers if ((now - _lastBeaconResponse) >= 2500) { // limit rate of responses _lastBeaconResponse = now; -- cgit v1.2.3 From 8a2ff0b31ea5df159b90640ccb9b86e8cb9040bc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 3 Feb 2017 19:47:00 -0800 Subject: Actual documentation. --- doc/MANUAL.md | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++------ node/README.md | 2 +- 2 files changed, 95 insertions(+), 12 deletions(-) (limited to 'node') diff --git a/doc/MANUAL.md b/doc/MANUAL.md index baca77ba..abaeef26 100644 --- a/doc/MANUAL.md +++ b/doc/MANUAL.md @@ -3,22 +3,18 @@ ZeroTier Planetary Switch Users Guide This manual describes the design and operation of ZeroTier and its associated services, apps, and libraries. Its intended audience includes IT professionals, network administrators, information security experts, and developers. -[ZeroTier Central](https://my.zerotier.com/), our enterprise web UI, has its own documentation that can be accessed through its interface. The high level ZeroTier rule description language used in Central also has its own documentation. This guide describes the core ZeroTier engine as deployed at endpoints as well as the deployment and operation of network controllers themselves. +[ZeroTier Central](https://my.zerotier.com/), our enterprise web UI, has its own help and documentation that can be accessed through its interface. ## Table of Contents 1. [Introduction](#1) 2. [How it Works](#2) 1. [VL1: The ZeroTier Peer to Peer Network](#2_1) - 1. [ZeroTier Addressing System](#2_1_1) - 2. [Encryption and Authentication](#2_1_2) - 3. [Trust Relationships](#2_1_3) - 4. [Global Roots and WAN Peer Discovery](#2_1_4) - 5. [Direct vs. Relayed Communication](#2_1_5) - 6. [LAN Peer Discovery](#2_1_6) - 7. [Federated Roots ("Moons")](#2_1_7) - 8. [Trusted Paths for Fast Local SDN](#2_1_8) - 9. [Troubleshooting Connectivity Problems](#2_1_9) + 1. [Network Topology and Peer Discovery](#2_1_1) + 2. [Addressing](#2_1_2) + 3. [Encryption and Authentication](#2_1_3) + 4. [Trusted Paths for Fast Local SDN](#2_1_4) + 5. [Troubleshooting Connectivity Problems](#2_1_5) 2. [VL2: The Ethernet Virtualization Layer](#2_2) 1. [Networks Identifiers and Controllers](#2_2_1) 2. [Multicast](#2_2_2) @@ -65,7 +61,7 @@ This manual describes the design and operation of ZeroTier and its associated se 2. [ZeroTier One on Linux or BSD Powered IoT Devices](#6_5_2) 7. [For Developers: Connecting IoT Devices and Apps](#7) 1. [ZeroTier SDK for Apps](#7_1) - 2. [ZeroTier Core](#7_2) + 2. [ZeroTier Network Hypervisor Core](#7_2) 1. [Code Layout and Design](#7_2_1) 2. [Building and Using](#7_2_2) 7. [Licensing](#8) @@ -74,3 +70,90 @@ This manual describes the design and operation of ZeroTier and its associated se ## **1.** Introduction +ZeroTier is a smart Ethernet switch for planet Earth. + +When the world is a single data center VPN, SDN, SD-WAN, and application peer to peer networking converge. The vast byzantine complexity of managing all these systems independently largely disappears. We've re-thought networking from first principles to deliver the flat end-to-end simplicity of the original pre-NAT pre-mobility Internet but in a way that meets the security, privacy, and mobility requirements of the 21st century. + +This guide is written for users with at least an intermediate understanding of topics like Ethernet and TCP/IP networking. It explains ZeroTier's design and use in considerable detail. Most users with sufficient IT expertise to configure a router or firewall will *not* need this guide to deploy ZeroTier for simple use cases. Indeed we've built a substantial user base prior to its publication. + +So before reading all this you might want to just try installing ZeroTier on a few things and creating a network. Come back when you want to understand what's happening or when you need to make use of more advanced features like rules, capabilities, federation, or clustering. + +## **2.** How it Works + +This section explains how ZeroTier's network hypervisor works. It's not required reading to operate ZeroTier for all but the most advanced deployments, but understanding how things work is always helpful if you ever need to troubleshoot anything. + +ZeroTier is comprised of two closely coupled but conceptually distinct layers [in the OSI model](https://en.wikipedia.org/wiki/OSI_model) sense: a virtual "wire" layer called VL1 that moves data around and a virtual switched Ethernet layer called VL2 to provide devices and apps with a familiar interface. Since almost any protocol can be carried over Ethernet, emulating standard Ethernet behavior maximizes versatility. + +### **2.1.** VL1: The ZeroTier Peer to Peer Network + +To build a planetary data center we first had to begin with the wiring. Tunneling into the Earth's core and putting a giant wire closet down there wasn't an option, so we decided to use software to build virtual wires over the existing Internet instead. + +In conventional networks L1 (OSI layer 1) refers to the actual CAT5/CAT6 cables or wireless radio channels over which data is carried and the physical transciever chips that modulate and demodulate it. VL1 is a peer to peer network that does the same thing by using encryption, authentication, and a lot of networking tricks to create virtual wires as needed. + +### **2.1.1.** Network Topology and Peer Discovery + +VL1's persistent structure is a hierarchical tree similar to DNS, but its leaves make direct ephemeral connections to one another on demand. At the base of the tree resides a pool of equal and fully redundant *roots* whose function is closely analogous to that of [DNS root name servers](https://en.wikipedia.org/wiki/Root_name_server). + +Roots run the same software as regular endpoints but reside at fast stable locations on the network and are designated as such by a *world definition*. There are two kinds of world definitions: a *planet* and a *moon*. The ZeroTier protocol contains a secure mechanism allowing world definitions to be updated in band. + +There is only one planet. Earth's root servers are operated by ZeroTier, Inc. as a free service. Their presence defines and unifies the global data center where we all reside. + +Users can create "moons." These nominate additional roots for redundancy or performance. The most common reasons for doing this are to eliminate hard dependency on ZeroTier's third party infrastructure or to designate local roots inside your building or cloud so ZeroTier can work without a connection to the Internet. Moons are by no means required and most of our users get by just fine without them. + +When peers start out they have no direct links to one another, only upstream to roots. Every peer on VL1 possesses a globally unique address, but unlike IP addresses these are opaque cryptographic identifiers that encode no routing information. To communicate peers first send packets "up" the tree, and as these packets traverse the network they trigger the opportunistic creation of direct links along the way. The tree is constantly trying to "collapse itself" to optimize itself to the pattern of traffic it is carrying. + +In the simplest case using only global roots, an initial connection setup between peers A and B goes like this: + +1. A wants to send a packet to B, but since it has no direct path it sends it upstream to R (a root). +2. R *does* have a direct link to B so it forwards the packet there. +3. R also sends a message called *RENDEZVOUS* to A containing hints about how it might reach B, and to B informing it how it might reach A. We call this "transport triggered link provisioning." +4. A and B get *RENDEZVOUS* and attempt to send test messages to each other, possibly accomplishing [hole punching](https://en.wikipedia.org/wiki/UDP_hole_punching) of any NATs or stateful firewalls that happen to be in the way. If this works a direct link is established and packets no longer need to take the scenic route. + +If R is a federated root the same process occurs, but possibly with additional steps. + +VL1 provides instant always-on virtual L1 connectivity between all devices in the world. Indirect paths are automatically and transparently upgraded to direct paths whenever possible, and if a direct path is lost ZeroTier falls back to indirect communication and the process begins again. + +If a direct path can never be established, indirect communication can continue forever with direct connection attempts also continuing indefinitely on a periodic basis. The protocol also contains other facilities for direct connectivity establishment such as LAN peer discovery, port prediction to traverse IPv4 symmetric NATs, and explicit endpoint advertisement that can be coupled with port mapping using uPnP or NAT-PMP if available. VL1 is persistent and determined when it comes to finding the most efficient way to move data. + +This is not a wholly unique design. It shares features in common with STUN/ICE, SIP, and other protocols. The most novel aspect may be the simplification achieved through lazy transport triggered link provisioning. This trades the [complicated state machines of STUN/ICE](https://www.pkcsecurity.com/untangling-webrtc-flow.html) for a stateless algorithm with implicit empirical parameters. It also eliminates asymmetry. As mentioned above, roots run the same code as regular nodes. + +*[A blog post from 2014 by ZeroTier's original author explains some of the reasoning process that led to this design.](http://adamierymenko.com/decentralization-i-want-to-believe/)* + +### **2.1.2.** Addressing + +Every device (a "device" can be anything from a laptop to an app) is identified on VL1 by a 40-bit (10 hex digit) unique *ZeroTier address*. These are the addresses used to address packets in the process described in 2.1.1 above. + +These addresses are computed from the public portion of a public/private key pair. An address along with its public key is called an *identity*. If you look at the home directory of a running ZeroTier instance you will see `identity.public` and `identity.secret`. + +When ZeroTier starts for the first time it generates a new key pair and a new identity. It then attempts to advertise it upstream to the network. In the very unlikely event that the identity's 40-bit unique address is taken, it discards it and generates another. + +Identities are claimed on a first come first serve basis and currently expire from global roots after 60 days of inactivity. If a long-dormant device returns it may re-claim its identity unless its address has been taken in the meantime (again, highly unlikely). + +The address derivation algorithm used to compute addresses from public keys imposes a computational cost barrier against the intentional generation of a collision. Currently it would take approximately 10,000 CPU-years to do so (assuming e.g. a 3ghz Intel core). This is expensive but not impossible, but it's only the first line of defense. After generating a collision an attacker would then have to compromise all upstream nodes and replace the address's cached identity, not to mention also doing the same for peers that have seen the target identity recently. + +In addition to assisting with communication, upstream nodes also act as identity caches. If the identity corresponding to an address is not known a peer may request it by sending a message called *WHOIS* upstream. + +### **2.1.3.** Encryption and Authentication + +If you don't know much about cryptography you can safely skip this section. **TL;DR: packets are end-to-end encrypted and can't be read by roots or anyone else, and we use modern 256-bit crypto in ways recommended by the professional cryptographers that created it.** + +Asymmetric public key encryption is [Curve25519/Ed25519](https://en.wikipedia.org/wiki/Curve25519), a 256-bit elliptic curve variant. + +Every VL1 packet is encrypted end to end using (as of the current version) 256-bit [Salsa20](https://ianix.com/pub/salsa20-deployment.html) and authenticated using the [Poly1305](https://en.wikipedia.org/wiki/Poly1305) message authentication (MAC) algorithm. + +MAC is computed after encryption [(encrypt-then-MAC)](https://tonyarcieri.com/all-the-crypto-code-youve-ever-written-is-probably-broken) and the cipher/MAC composition used is identical to the [NaCl reference implementation](https://nacl.cr.yp.to). + +As of today we do not implement [forward secrecy](https://en.wikipedia.org/wiki/Forward_secrecy) or other stateful cryptographic features in VL1. We don't do this for the sake of simplicity, reliability, and code footprint, and because frequently changing state makes features like clustering and fail-over much harder to implement. See [our discussion on GitHub](https://github.com/zerotier/ZeroTierOne/issues/204). + +For those who have very high security needs and want forward secrecy, we currently recommend the use of encrypted protocols such as SSH and SSL over ZeroTier. Not only do these provide forward secrecy, the use of multiple layers of encryption in this way provides excellent [defense in depth](https://en.wikipedia.org/wiki/Defense_in_depth_%28computing%29). The computational cost of this additional crypto is typically small, and the benefit can potentially be large. All software can contain bugs, but multiple layers of protection means that discovery of a catastrophic bug in any one layer does not result in compromise of your entire system. We recommend the same for authentication. While ZeroTier VL2 provides certificate-based network boundary enforcement, we do not recommend that users rely solely on this for access control to critical systems. It is always good to use more than one security measure whenever practical. + +### **2.1.4.** Trusted Paths for Fast Local SDN + +To support the use of ZeroTier as a high performance SDN/NFV protocol over physically secure networks the protocol supports a feature called *trusted paths*. It is possible to configure all ZeroTier devices on a given network to skip encryption and authentication for traffic over a designated physical path. This can cut CPU use noticably in high traffic scenarios but at the expense of effectively all transport security over the configured trusted backplane. + +Trusted paths do not prevent communication with devices elsewhere, since traffic over other paths will be encrypted and authenticated normally. + +We don't recommend the use of this feature unless you really need the performance and you know what you're doing. Extra security is never a bad thing. We also recommend thinking carefully before disabling transport security on a cloud private network. Larger cloud providers such as Amazon and Azure tend to provide good network segregation but many less costly providers offer private networks that are "party lines." For these the encryption and authentication provided by ZeroTier is very desirable. In fact, we have a few users using ZeroTier exactly for this reason. + +### **2.1.5.** Troubleshooting Connectivity Problems + diff --git a/node/README.md b/node/README.md index 01378c75..1728400c 100644 --- a/node/README.md +++ b/node/README.md @@ -1,4 +1,4 @@ -ZeroTier Virtual Switch Core +ZeroTier Network Hypervisor Core ====== This directory contains the *real* ZeroTier: a completely OS-independent global virtual Ethernet switch engine. This is where the magic happens. -- cgit v1.2.3 From dcb1233b0d5478f2f544a99cf24314155e711050 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 3 Feb 2017 23:54:02 -0800 Subject: Slight refactor to RENEDEZVOUS sending code for federation. --- node/Cluster.cpp | 4 +- node/Peer.cpp | 2 +- node/Peer.hpp | 2 +- node/Switch.cpp | 171 ++++++++++++++++++++++--------------------------------- node/Switch.hpp | 2 - 5 files changed, 72 insertions(+), 109 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 356a0887..00122402 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -400,7 +400,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) SharedPtr localPeer(RR->topology->getPeerNoCache(localPeerAddress)); if ((localPeer)&&(numRemotePeerPaths > 0)) { InetAddress bestLocalV4,bestLocalV6; - localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6); + localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6); InetAddress bestRemoteV4,bestRemoteV6; for(unsigned int i=0;i fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); if (fromPeer) - fromPeer->getBestActiveAddresses(now,v4,v6); + fromPeer->getRendezvousAddresses(now,v4,v6); } uint8_t addrCount = 0; if (v4) diff --git a/node/Peer.cpp b/node/Peer.cpp index 50135b9f..129c2437 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -446,7 +446,7 @@ void Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uin } } -void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const +void Peer::getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const { Mutex::Lock _l(_paths_m); diff --git a/node/Peer.hpp b/node/Peer.hpp index 06abde3f..bbe13a2e 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -208,7 +208,7 @@ public: * @param v4 Result parameter to receive active IPv4 address, if any * @param v6 Result parameter to receive active IPv6 address, if any */ - void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; + void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; /** * @param now Current time diff --git a/node/Switch.cpp b/node/Switch.cpp index 2ab3ccfc..b53466cf 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -64,10 +64,6 @@ Switch::Switch(const RuntimeEnvironment *renv) : { } -Switch::~Switch() -{ -} - void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len) { try { @@ -221,7 +217,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from if (packet.hops() < ZT_RELAY_MAX_HOPS) { #ifdef ZT_ENABLE_CLUSTER - if (source != RR->identity.address()) + if (source != RR->identity.address()) // don't increment hops for cluster frontplane relays packet.incrementHops(); #else packet.incrementHops(); @@ -229,13 +225,74 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from SharedPtr relayTo = RR->topology->getPeer(destination); if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { - if (source != RR->identity.address()) { - Mutex::Lock _l(_lastUniteAttempt_m); - uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; - if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) { - luts = now; - _unite(source,destination); + if (source != RR->identity.address()) { // don't send RENDEZVOUS for cluster frontplane relays + + bool shouldUnite; + { + Mutex::Lock _l(_lastUniteAttempt_m); + uint64_t &lastUniteAt = _lastUniteAttempt[_LastUniteKey(source,destination)]; + shouldUnite = ((now - lastUniteAt) >= ZT_MIN_UNITE_INTERVAL); + if (shouldUnite) + lastUniteAt = now; } + + if (shouldUnite) { + const InetAddress *hintToSource = (InetAddress *)0; + const InetAddress *hintToDest = (InetAddress *)0; + + InetAddress destV4,destV6; + InetAddress sourceV4,sourceV6; + relayTo->getRendezvousAddresses(now,destV4,destV6); + + const SharedPtr sourcePeer(RR->topology->getPeer(source)); + if (sourcePeer) { + sourcePeer->getRendezvousAddresses(now,sourceV4,sourceV6); + if ((destV6)&&(sourceV6)) { + hintToSource = &destV6; + hintToDest = &sourceV6; + } else if ((destV4)&&(sourceV4)) { + hintToSource = &destV4; + hintToDest = &sourceV4; + } + + if ((hintToSource)&&(hintToDest)) { + TRACE(">> RENDEZVOUS: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); + unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for obscure NAT-t reasons + const unsigned int completed = alt + 2; + while (alt != completed) { + if ((alt & 1) == 0) { + Packet outp(source,RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((uint8_t)0); + destination.appendTo(outp); + outp.append((uint16_t)hintToSource->port()); + if (hintToSource->ss_family == AF_INET6) { + outp.append((uint8_t)16); + outp.append(hintToSource->rawIpData(),16); + } else { + outp.append((uint8_t)4); + outp.append(hintToSource->rawIpData(),4); + } + send(outp,true); + } else { + Packet outp(destination,RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((uint8_t)0); + source.appendTo(outp); + outp.append((uint16_t)hintToDest->port()); + if (hintToDest->ss_family == AF_INET6) { + outp.append((uint8_t)16); + outp.append(hintToDest->rawIpData(),16); + } else { + outp.append((uint8_t)4); + outp.append(hintToDest->rawIpData(),4); + } + send(outp,true); + } + ++alt; + } + } + } + } + } } else { #ifdef ZT_ENABLE_CLUSTER @@ -824,96 +881,4 @@ bool Switch::_trySend(Packet &packet,bool encrypt) return true; } -bool Switch::_unite(const Address &p1,const Address &p2) -{ - if ((p1 == RR->identity.address())||(p2 == RR->identity.address())) - return false; - - const uint64_t now = RR->node->now(); - InetAddress *p1a = (InetAddress *)0; - InetAddress *p2a = (InetAddress *)0; - InetAddress p1v4,p1v6,p2v4,p2v6,uv4,uv6; - { - const SharedPtr p1p(RR->topology->getPeer(p1)); - const SharedPtr p2p(RR->topology->getPeer(p2)); - if ((!p1p)&&(!p2p)) return false; - if (p1p) p1p->getBestActiveAddresses(now,p1v4,p1v6); - if (p2p) p2p->getBestActiveAddresses(now,p2v4,p2v6); - } - if ((p1v6)&&(p2v6)) { - p1a = &p1v6; - p2a = &p2v6; - } else if ((p1v4)&&(p2v4)) { - p1a = &p1v4; - p2a = &p2v4; - } else { - SharedPtr upstream(RR->topology->getUpstreamPeer()); - if (!upstream) - return false; - upstream->getBestActiveAddresses(now,uv4,uv6); - if ((p1v6)&&(uv6)) { - p1a = &p1v6; - p2a = &uv6; - } else if ((p1v4)&&(uv4)) { - p1a = &p1v4; - p2a = &uv4; - } else if ((p2v6)&&(uv6)) { - p1a = &p2v6; - p2a = &uv6; - } else if ((p2v4)&&(uv4)) { - p1a = &p2v4; - p2a = &uv4; - } else return false; - } - - TRACE("unite: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); - - /* Tell P1 where to find P2 and vice versa, sending the packets to P1 and - * P2 in randomized order in terms of which gets sent first. This is done - * since in a few cases NAT-t can be sensitive to slight timing differences - * in terms of when the two peers initiate. Normally this is accounted for - * by the nearly-simultaneous RENDEZVOUS kickoff from the relay, but - * given that relay are hosted on cloud providers this can in some - * cases have a few ms of latency between packet departures. By randomizing - * the order we make each attempted NAT-t favor one or the other going - * first, meaning if it doesn't succeed the first time it might the second - * and so forth. */ - unsigned int alt = (unsigned int)RR->node->prng() & 1; - const unsigned int completed = alt + 2; - while (alt != completed) { - if ((alt & 1) == 0) { - // Tell p1 where to find p2. - Packet outp(p1,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((unsigned char)0); - p2.appendTo(outp); - outp.append((uint16_t)p2a->port()); - if (p2a->isV6()) { - outp.append((unsigned char)16); - outp.append(p2a->rawIpData(),16); - } else { - outp.append((unsigned char)4); - outp.append(p2a->rawIpData(),4); - } - send(outp,true); - } else { - // Tell p2 where to find p1. - Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((unsigned char)0); - p1.appendTo(outp); - outp.append((uint16_t)p1a->port()); - if (p1a->isV6()) { - outp.append((unsigned char)16); - outp.append(p1a->rawIpData(),16); - } else { - outp.append((unsigned char)4); - outp.append(p1a->rawIpData(),4); - } - send(outp,true); - } - ++alt; // counts up and also flips LSB - } - - return true; -} - } // namespace ZeroTier diff --git a/node/Switch.hpp b/node/Switch.hpp index 422f6c8e..ce1c40b3 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -55,7 +55,6 @@ class Switch : NonCopyable { public: Switch(const RuntimeEnvironment *renv); - ~Switch(); /** * Called when a packet is received from the real network @@ -127,7 +126,6 @@ public: private: Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); bool _trySend(Packet &packet,bool encrypt); // packet is modified if return is true - bool _unite(const Address &p1,const Address &p2); const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; -- cgit v1.2.3 From d9e4ba1280bc5aec5d4cc04c4c495ac9c923563d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 4 Feb 2017 00:04:44 -0800 Subject: Eliminate a little copypasta. --- node/Switch.cpp | 130 ++++++++++++++++++++++++++------------------------------ node/Switch.hpp | 1 + 2 files changed, 61 insertions(+), 70 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index b53466cf..9fcd379f 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -225,87 +225,66 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from SharedPtr relayTo = RR->topology->getPeer(destination); if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { - if (source != RR->identity.address()) { // don't send RENDEZVOUS for cluster frontplane relays - - bool shouldUnite; - { - Mutex::Lock _l(_lastUniteAttempt_m); - uint64_t &lastUniteAt = _lastUniteAttempt[_LastUniteKey(source,destination)]; - shouldUnite = ((now - lastUniteAt) >= ZT_MIN_UNITE_INTERVAL); - if (shouldUnite) - lastUniteAt = now; - } - - if (shouldUnite) { - const InetAddress *hintToSource = (InetAddress *)0; - const InetAddress *hintToDest = (InetAddress *)0; - - InetAddress destV4,destV6; - InetAddress sourceV4,sourceV6; - relayTo->getRendezvousAddresses(now,destV4,destV6); - - const SharedPtr sourcePeer(RR->topology->getPeer(source)); - if (sourcePeer) { - sourcePeer->getRendezvousAddresses(now,sourceV4,sourceV6); - if ((destV6)&&(sourceV6)) { - hintToSource = &destV6; - hintToDest = &sourceV6; - } else if ((destV4)&&(sourceV4)) { - hintToSource = &destV4; - hintToDest = &sourceV4; - } + if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { // don't send RENDEZVOUS for cluster frontplane relays + const InetAddress *hintToSource = (InetAddress *)0; + const InetAddress *hintToDest = (InetAddress *)0; + + InetAddress destV4,destV6; + InetAddress sourceV4,sourceV6; + relayTo->getRendezvousAddresses(now,destV4,destV6); + + const SharedPtr sourcePeer(RR->topology->getPeer(source)); + if (sourcePeer) { + sourcePeer->getRendezvousAddresses(now,sourceV4,sourceV6); + if ((destV6)&&(sourceV6)) { + hintToSource = &destV6; + hintToDest = &sourceV6; + } else if ((destV4)&&(sourceV4)) { + hintToSource = &destV4; + hintToDest = &sourceV4; + } - if ((hintToSource)&&(hintToDest)) { - TRACE(">> RENDEZVOUS: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); - unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for obscure NAT-t reasons - const unsigned int completed = alt + 2; - while (alt != completed) { - if ((alt & 1) == 0) { - Packet outp(source,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((uint8_t)0); - destination.appendTo(outp); - outp.append((uint16_t)hintToSource->port()); - if (hintToSource->ss_family == AF_INET6) { - outp.append((uint8_t)16); - outp.append(hintToSource->rawIpData(),16); - } else { - outp.append((uint8_t)4); - outp.append(hintToSource->rawIpData(),4); - } - send(outp,true); + if ((hintToSource)&&(hintToDest)) { + TRACE(">> RENDEZVOUS: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); + unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for obscure NAT-t reasons + const unsigned int completed = alt + 2; + while (alt != completed) { + if ((alt & 1) == 0) { + Packet outp(source,RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((uint8_t)0); + destination.appendTo(outp); + outp.append((uint16_t)hintToSource->port()); + if (hintToSource->ss_family == AF_INET6) { + outp.append((uint8_t)16); + outp.append(hintToSource->rawIpData(),16); + } else { + outp.append((uint8_t)4); + outp.append(hintToSource->rawIpData(),4); + } + send(outp,true); + } else { + Packet outp(destination,RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((uint8_t)0); + source.appendTo(outp); + outp.append((uint16_t)hintToDest->port()); + if (hintToDest->ss_family == AF_INET6) { + outp.append((uint8_t)16); + outp.append(hintToDest->rawIpData(),16); } else { - Packet outp(destination,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((uint8_t)0); - source.appendTo(outp); - outp.append((uint16_t)hintToDest->port()); - if (hintToDest->ss_family == AF_INET6) { - outp.append((uint8_t)16); - outp.append(hintToDest->rawIpData(),16); - } else { - outp.append((uint8_t)4); - outp.append(hintToDest->rawIpData(),4); - } - send(outp,true); + outp.append((uint8_t)4); + outp.append(hintToDest->rawIpData(),4); } - ++alt; + send(outp,true); } + ++alt; } } } - } } else { #ifdef ZT_ENABLE_CLUSTER if ((RR->cluster)&&(source != RR->identity.address())) { - bool shouldUnite; - { - Mutex::Lock _l(_lastUniteAttempt_m); - uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)]; - shouldUnite = ((now - luts) >= ZT_MIN_UNITE_INTERVAL); - if (shouldUnite) - luts = now; - } - RR->cluster->relayViaCluster(source,destination,packet.data(),packet.size(),shouldUnite); + RR->cluster->relayViaCluster(source,destination,packet.data(),packet.size(),_shouldUnite(now,source,destination)); return; } #endif @@ -747,6 +726,17 @@ unsigned long Switch::doTimerTasks(uint64_t now) return nextDelay; } +bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address &destination) +{ + Mutex::Lock _l(_lastUniteAttempt_m); + uint64_t &ts = _lastUniteAttempt[_LastUniteKey(source,destination)]; + if ((now - ts) >= ZT_MIN_UNITE_INTERVAL) { + ts = now; + return true; + } + return false; +} + Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) { SharedPtr upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); diff --git a/node/Switch.hpp b/node/Switch.hpp index ce1c40b3..9245c036 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -124,6 +124,7 @@ public: unsigned long doTimerTasks(uint64_t now); private: + bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination); Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); bool _trySend(Packet &packet,bool encrypt); // packet is modified if return is true -- cgit v1.2.3 From 31db768e4d4c2815d2be0493b2c76ea5f5edbffa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 4 Feb 2017 00:23:31 -0800 Subject: A bit of code cleanup. --- controller/EmbeddedNetworkController.cpp | 2 +- doc/MANUAL.md | 2 - node/Address.hpp | 77 +++++++++----------------------- node/Identity.cpp | 2 +- node/NetworkConfig.cpp | 2 +- node/Switch.cpp | 3 +- service/OneService.cpp | 2 +- 7 files changed, 25 insertions(+), 65 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3c13e7ce..e7bcdd07 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1690,7 +1690,7 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid } if (OSUtils::jsonBool(member["activeBridge"],false)) { - nmi.activeBridges.insert(OSUtils::jsonString(member["id"],"0000000000")); + nmi.activeBridges.insert(Address(Utils::hexStrToU64(OSUtils::jsonString(member["id"],"0000000000").c_str()))); } if (member.count("ipAssignments")) { diff --git a/doc/MANUAL.md b/doc/MANUAL.md index abaeef26..8af8fc2d 100644 --- a/doc/MANUAL.md +++ b/doc/MANUAL.md @@ -109,8 +109,6 @@ In the simplest case using only global roots, an initial connection setup betwee 3. R also sends a message called *RENDEZVOUS* to A containing hints about how it might reach B, and to B informing it how it might reach A. We call this "transport triggered link provisioning." 4. A and B get *RENDEZVOUS* and attempt to send test messages to each other, possibly accomplishing [hole punching](https://en.wikipedia.org/wiki/UDP_hole_punching) of any NATs or stateful firewalls that happen to be in the way. If this works a direct link is established and packets no longer need to take the scenic route. -If R is a federated root the same process occurs, but possibly with additional steps. - VL1 provides instant always-on virtual L1 connectivity between all devices in the world. Indirect paths are automatically and transparently upgraded to direct paths whenever possible, and if a direct path is lost ZeroTier falls back to indirect communication and the process begins again. If a direct path can never be established, indirect communication can continue forever with direct connection attempts also continuing indefinitely on a periodic basis. The protocol also contains other facilities for direct connectivity establishment such as LAN peer discovery, port prediction to traverse IPv4 symmetric NATs, and explicit endpoint advertisement that can be coupled with port mapping using uPnP or NAT-PMP if available. VL1 is persistent and determined when it comes to finding the most efficient way to move data. diff --git a/node/Address.hpp b/node/Address.hpp index 9bf5605a..4a5883b0 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -38,57 +38,26 @@ namespace ZeroTier { class Address { public: - Address() - throw() : - _a(0) - { - } - - Address(const Address &a) - throw() : - _a(a._a) - { - } - - Address(uint64_t a) - throw() : - _a(a & 0xffffffffffULL) - { - } - - Address(const char *s) - throw() - { - unsigned char foo[ZT_ADDRESS_LENGTH]; - setTo(foo,Utils::unhex(s,foo,ZT_ADDRESS_LENGTH)); - } - - Address(const std::string &s) - throw() - { - unsigned char foo[ZT_ADDRESS_LENGTH]; - setTo(foo,Utils::unhex(s.c_str(),foo,ZT_ADDRESS_LENGTH)); - } + Address() : _a(0) {} + Address(const Address &a) : _a(a._a) {} + Address(uint64_t a) : _a(a & 0xffffffffffULL) {} /** * @param bits Raw address -- 5 bytes, big-endian byte order * @param len Length of array */ Address(const void *bits,unsigned int len) - throw() { setTo(bits,len); } inline Address &operator=(const Address &a) - throw() { _a = a._a; return *this; } inline Address &operator=(const uint64_t a) - throw() { _a = (a & 0xffffffffffULL); return *this; @@ -99,7 +68,6 @@ public: * @param len Length of array */ inline void setTo(const void *bits,unsigned int len) - throw() { if (len < ZT_ADDRESS_LENGTH) { _a = 0; @@ -119,7 +87,6 @@ public: * @param len Length of array */ inline void copyTo(void *bits,unsigned int len) const - throw() { if (len < ZT_ADDRESS_LENGTH) return; @@ -138,7 +105,6 @@ public: */ template inline void appendTo(Buffer &b) const - throw(std::out_of_range) { unsigned char *p = (unsigned char *)b.appendField(ZT_ADDRESS_LENGTH); *(p++) = (unsigned char)((_a >> 32) & 0xff); @@ -152,7 +118,6 @@ public: * @return Integer containing address (0 to 2^40) */ inline uint64_t toInt() const - throw() { return _a; } @@ -161,7 +126,6 @@ public: * @return Hash code for use with Hashtable */ inline unsigned long hashCode() const - throw() { return (unsigned long)_a; } @@ -188,12 +152,12 @@ public: /** * @return True if this address is not zero */ - inline operator bool() const throw() { return (_a != 0); } + inline operator bool() const { return (_a != 0); } /** * Set to null/zero */ - inline void zero() throw() { _a = 0; } + inline void zero() { _a = 0; } /** * Check if this address is reserved @@ -205,7 +169,6 @@ public: * @return True if address is reserved and may not be used */ inline bool isReserved() const - throw() { return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); } @@ -214,21 +177,21 @@ public: * @param i Value from 0 to 4 (inclusive) * @return Byte at said position (address interpreted in big-endian order) */ - inline unsigned char operator[](unsigned int i) const throw() { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); } - - inline bool operator==(const uint64_t &a) const throw() { return (_a == (a & 0xffffffffffULL)); } - inline bool operator!=(const uint64_t &a) const throw() { return (_a != (a & 0xffffffffffULL)); } - inline bool operator>(const uint64_t &a) const throw() { return (_a > (a & 0xffffffffffULL)); } - inline bool operator<(const uint64_t &a) const throw() { return (_a < (a & 0xffffffffffULL)); } - inline bool operator>=(const uint64_t &a) const throw() { return (_a >= (a & 0xffffffffffULL)); } - inline bool operator<=(const uint64_t &a) const throw() { return (_a <= (a & 0xffffffffffULL)); } - - inline bool operator==(const Address &a) const throw() { return (_a == a._a); } - inline bool operator!=(const Address &a) const throw() { return (_a != a._a); } - inline bool operator>(const Address &a) const throw() { return (_a > a._a); } - inline bool operator<(const Address &a) const throw() { return (_a < a._a); } - inline bool operator>=(const Address &a) const throw() { return (_a >= a._a); } - inline bool operator<=(const Address &a) const throw() { return (_a <= a._a); } + inline unsigned char operator[](unsigned int i) const { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); } + + inline bool operator==(const uint64_t &a) const { return (_a == (a & 0xffffffffffULL)); } + inline bool operator!=(const uint64_t &a) const { return (_a != (a & 0xffffffffffULL)); } + inline bool operator>(const uint64_t &a) const { return (_a > (a & 0xffffffffffULL)); } + inline bool operator<(const uint64_t &a) const { return (_a < (a & 0xffffffffffULL)); } + inline bool operator>=(const uint64_t &a) const { return (_a >= (a & 0xffffffffffULL)); } + inline bool operator<=(const uint64_t &a) const { return (_a <= (a & 0xffffffffffULL)); } + + inline bool operator==(const Address &a) const { return (_a == a._a); } + inline bool operator!=(const Address &a) const { return (_a != a._a); } + inline bool operator>(const Address &a) const { return (_a > a._a); } + inline bool operator<(const Address &a) const { return (_a < a._a); } + inline bool operator>=(const Address &a) const { return (_a >= a._a); } + inline bool operator<=(const Address &a) const { return (_a <= a._a); } private: uint64_t _a; diff --git a/node/Identity.cpp b/node/Identity.cpp index c47805d9..05b70873 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -160,7 +160,7 @@ bool Identity::fromString(const char *str) for(char *f=Utils::stok(tmp,":",&saveptr);(f);f=Utils::stok((char *)0,":",&saveptr)) { switch(fno++) { case 0: - _address = Address(f); + _address = Address(Utils::hexStrToU64(f)); if (_address.isReserved()) return false; break; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 6acc48ea..2f356b15 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -258,7 +258,7 @@ bool NetworkConfig::fromDictionary(const Dictionary 0) { char *saveptr = (char *)0; for(char *f=Utils::stok(tmp2,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) { - this->addSpecialist(Address(f),ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + this->addSpecialist(Address(Utils::hexStrToU64(f)),ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); } } #else diff --git a/node/Switch.cpp b/node/Switch.cpp index 9fcd379f..a769faea 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -224,7 +224,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif SharedPtr relayTo = RR->topology->getPeer(destination); - if ((relayTo)&&((relayTo->sendDirect(packet.data(),packet.size(),now,false)))) { + if ((relayTo)&&(relayTo->sendDirect(packet.data(),packet.size(),now,false))) { if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { // don't send RENDEZVOUS for cluster frontplane relays const InetAddress *hintToSource = (InetAddress *)0; const InetAddress *hintToDest = (InetAddress *)0; @@ -245,7 +245,6 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } if ((hintToSource)&&(hintToDest)) { - TRACE(">> RENDEZVOUS: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str()); unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for obscure NAT-t reasons const unsigned int completed = alt + 2; while (alt != completed) { diff --git a/service/OneService.cpp b/service/OneService.cpp index 49c5f4a0..9a1503e5 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -926,7 +926,7 @@ public: for(json::iterator v(virt.begin());v!=virt.end();++v) { const std::string nstr = v.key(); if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) { - const Address ztaddr(nstr.c_str()); + const Address ztaddr(Utils::hexStrToU64(nstr.c_str())); if (ztaddr) { const uint64_t ztaddr2 = ztaddr.toInt(); std::vector &v4h = _v4Hints[ztaddr2]; -- cgit v1.2.3 From beb642faa58bb3c2c283a068e6de942bfad2c314 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 4 Feb 2017 10:21:31 -0800 Subject: Stub out CAN_REACH. --- node/Constants.hpp | 5 +++++ node/IncomingPacket.cpp | 7 +++++-- node/Packet.hpp | 29 +++++++++++++++++++++++++---- 3 files changed, 35 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index a73d4d89..ab6dfb32 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -214,6 +214,11 @@ */ #define ZT_RECEIVE_QUEUE_TIMEOUT (ZT_WHOIS_RETRY_DELAY * (ZT_MAX_WHOIS_RETRIES + 1)) +/** + * Maximum latency to allow for OK(HELLO) before packet is discarded + */ +#define ZT_HELLO_MAX_ALLOWABLE_LATENCY 60000 + /** * Maximum number of ZT hops allowed (this is not IP hops/TTL) * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c11b0377..cecbe2fa 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -413,7 +413,10 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p switch(inReVerb) { case Packet::VERB_HELLO: { - const unsigned int latency = std::min((unsigned int)(RR->node->now() - at(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)),(unsigned int)0xffff); + const uint64_t latency = RR->node->now() - at(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP); + if (latency > ZT_HELLO_MAX_ALLOWABLE_LATENCY) + return true; + const unsigned int vProto = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION]; const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION]; const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION]; @@ -445,7 +448,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); if (!hops()) - peer->addDirectLatencyMeasurment(latency); + peer->addDirectLatencyMeasurment((unsigned int)latency); peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if ((externalSurfaceAddress)&&(hops() == 0)) diff --git a/node/Packet.hpp b/node/Packet.hpp index 26e87af8..a5831c8d 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -536,7 +536,7 @@ public: * <[1] software major version> * <[1] software minor version> * <[2] software revision> - * <[8] timestamp for determining latench> + * <[8] timestamp for determining latency> * <[...] binary serialized identity (see Identity)> * <[1] destination address type> * [<[...] destination address to which packet was sent>] @@ -548,8 +548,9 @@ public: * [<[8] 64-bit timestamp of moon>] * [... additional moons ...] * - * This is the only message that ever must be sent in the clear, since it - * is used to push an identity to a new peer. + * Important security note: this message is sent in the clear as it + * contains the initial identity for key agreement. It can therefore + * contain no secrets or sensitive information. * * The destination address is the wire address to which this packet is * being sent, and in OK is *also* the destination address of the OK @@ -1058,7 +1059,27 @@ public: * ZeroTier, Inc. itself. We recommend making up random ones for your own * implementations. */ - VERB_USER_MESSAGE = 0x14 + VERB_USER_MESSAGE = 0x14, + + /** + * Announce that we can reach a particular address: + * <[1] protocol version> + * <[1] software major version> + * <[1] software minor version> + * <[2] software revision> + * <[...] binary serialized identity (see Identity)> + * <[1] 8-bit number of direct addresses where peer is reachable (if any)> + * [... serialized direct addresses ...] + * + * This message can be sent upstream to announce that we can reach a + * particular address. It can optionally report physical paths upstream + * to allow upstream peers to send RENDEZVOUS, but this may be omitted + * if it is not known or if endpoint address privacy is desired. + * + * The receiving peer should confirm this message by sending a message + * downstream and waiting for a reply. + */ + VERB_CAN_REACH = 0x15 }; /** -- cgit v1.2.3 From 3587aa1ea7573198168422be55511b16470fb33f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 4 Feb 2017 13:17:00 -0800 Subject: Add and send certificates of representation to tell people what our valid upstreams are. These are not used yet but will be needed for future privacy modes, etc. Also some cleanup. --- node/Capability.hpp | 9 +- node/CertificateOfRepresentation.hpp | 161 +++++++++++++++++++++++++++++++++++ node/Constants.hpp | 5 ++ node/IncomingPacket.cpp | 9 ++ node/Packet.hpp | 32 ++----- node/Peer.cpp | 7 +- node/Peer.hpp | 11 ++- node/Revocation.hpp | 2 + node/Tag.hpp | 14 +-- node/Topology.cpp | 11 +++ node/Topology.hpp | 21 +++++ 11 files changed, 247 insertions(+), 35 deletions(-) create mode 100644 node/CertificateOfRepresentation.hpp (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index 2c829ee5..ddbfd9ee 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -414,7 +414,14 @@ public: throw std::runtime_error("unterminated custody chain"); _custody[i].to = to; _custody[i].from.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - memcpy(_custody[i].signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + if (b[p++] == 1) { + if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) + throw std::runtime_error("invalid signature"); + p += 2; + memcpy(_custody[i].signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + } else { + p += 2 + b.template at(p); + } } p += 2 + b.template at(p); diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp new file mode 100644 index 00000000..7c239a96 --- /dev/null +++ b/node/CertificateOfRepresentation.hpp @@ -0,0 +1,161 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CERTIFICATEOFREPRESENTATION_HPP +#define ZT_CERTIFICATEOFREPRESENTATION_HPP + +#include "Constants.hpp" +#include "Address.hpp" +#include "C25519.hpp" +#include "Identity.hpp" +#include "Buffer.hpp" + +/** + * Maximum number of addresses allowed in a COR + */ +#define ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES ZT_MAX_UPSTREAMS + +namespace ZeroTier { + +class CertificateOfRepresentation +{ +public: + CertificateOfRepresentation() + { + memset(this,0,sizeof(CertificateOfRepresentation)); + } + + inline uint64_t timestamp() const { return _timestamp; } + inline const Address &representative(const unsigned int i) const { return _reps[i]; } + inline unsigned int repCount() const { return _repCount; } + + inline void clear() + { + memset(this,0,sizeof(CertificateOfRepresentation)); + } + + /** + * Add a representative if space remains + * + * @param r Representative to add + * @return True if representative was added + */ + inline bool addRepresentative(const Address &r) + { + if (_repCount < ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES) { + _reps[_repCount++] = r; + return true; + } + return false; + } + + /** + * Sign this COR with my identity + * + * @param myIdentity This node's identity + * @param ts COR timestamp for establishing new vs. old + */ + inline void sign(const Identity &myIdentity,const uint64_t ts) + { + _timestamp = ts; + Buffer tmp; + this->serialize(tmp,true); + _signature = myIdentity.sign(tmp.data(),tmp.size()); + } + + /** + * Verify this COR's signature + * + * @param senderIdentity Identity of sender of COR + * @return True if COR is valid + */ + inline bool verify(const Identity &senderIdentity) + { + try { + Buffer tmp; + this->serialize(tmp,true); + return senderIdentity.verify(tmp.data(),tmp.size(),_signature.data,ZT_C25519_SIGNATURE_LEN); + } catch ( ... ) { + return false; + } + } + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append((uint64_t)_timestamp); + b.append((uint16_t)_repCount); + for(unsigned int i=0;i<_repCount;++i) + _reps[i].appendTo(b); + + if (!forSign) { + b.append((uint8_t)1); // 1 == Ed25519 signature + b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); + b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); + } + + b.append((uint16_t)0); // size of any additional fields, currently 0 + + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + } + + template + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + clear(); + + unsigned int p = startAt; + + _timestamp = b.template at(p); p += 8; + const unsigned int rc = b.template at(p); p += 2; + for(unsigned int i=0;i ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES) ? ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES : rc; + + if (b[p++] == 1) { + if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); + p += ZT_C25519_SIGNATURE_LEN; + } else throw std::runtime_error("invalid signature"); + } else { + p += 2 + b.template at(p); + } + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + +private: + uint64_t _timestamp; + Address _reps[ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES]; + unsigned int _repCount; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/Constants.hpp b/node/Constants.hpp index ab6dfb32..be4eb475 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -226,6 +226,11 @@ */ #define ZT_RELAY_MAX_HOPS 3 +/** + * Maximum number of upstreams to use (far more than we should ever need) + */ +#define ZT_MAX_UPSTREAMS 64 + /** * Expire time for multicast 'likes' and indirect multicast memberships in ms */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index cecbe2fa..6b38c4ec 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -37,6 +37,7 @@ #include "Cluster.hpp" #include "Node.hpp" #include "CertificateOfMembership.hpp" +#include "CertificateOfRepresentation.hpp" #include "Capability.hpp" #include "Tag.hpp" #include "Revocation.hpp" @@ -445,6 +446,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p } } + // Handle COR if present (older versions don't send this) + if ((ptr + 2) <= size()) { + //const unsigned int corSize = at(ptr); ptr += 2; + ptr += 2; + CertificateOfRepresentation cor; + ptr += cor.deserialize(*this,ptr); + } + TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); if (!hops()) diff --git a/node/Packet.hpp b/node/Packet.hpp index a5831c8d..7d404b25 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -531,7 +531,7 @@ public: VERB_NOP = 0x00, /** - * Announcement of a node's existence: + * Announcement of a node's existence and vitals: * <[1] protocol version> * <[1] software major version> * <[1] software minor version> @@ -547,10 +547,12 @@ public: * [<[8] 64-bit world ID of moon>] * [<[8] 64-bit timestamp of moon>] * [... additional moons ...] + * <[2] 16-bit length of certificate of representation> + * [... certificate of representation ...] * - * Important security note: this message is sent in the clear as it - * contains the initial identity for key agreement. It can therefore - * contain no secrets or sensitive information. + * HELLO is sent in the clear, and therefore cannot contain anything + * secret or highly confidential. It should contain nothing that is + * not either public or easy to obtain via other means. * * The destination address is the wire address to which this packet is * being sent, and in OK is *also* the destination address of the OK @@ -1059,27 +1061,7 @@ public: * ZeroTier, Inc. itself. We recommend making up random ones for your own * implementations. */ - VERB_USER_MESSAGE = 0x14, - - /** - * Announce that we can reach a particular address: - * <[1] protocol version> - * <[1] software major version> - * <[1] software minor version> - * <[2] software revision> - * <[...] binary serialized identity (see Identity)> - * <[1] 8-bit number of direct addresses where peer is reachable (if any)> - * [... serialized direct addresses ...] - * - * This message can be sent upstream to announce that we can reach a - * particular address. It can optionally report physical paths upstream - * to allow upstream peers to send RENDEZVOUS, but this may be omitted - * if it is not known or if endpoint address privacy is desired. - * - * The receiving peer should confirm this message by sending a message - * downstream and waiting for a reply. - */ - VERB_CAN_REACH = 0x15 + VERB_USER_MESSAGE = 0x14 }; /** diff --git a/node/Peer.cpp b/node/Peer.cpp index 129c2437..bb6b945d 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,6 +38,7 @@ namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : + RR(renv), _lastReceive(0), _lastNontrivialReceive(0), _lastTriedMemorizedPath(0), @@ -50,7 +51,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastComRequestSent(0), _lastCredentialsReceived(0), _lastTrustEstablishedPacketReceived(0), - RR(renv), _remoteClusterOptimal4(0), _vProto(0), _vMajor(0), @@ -365,6 +365,11 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u outp.append((uint64_t)m->timestamp()); } + const unsigned int corSizeAt = outp.size(); + outp.addSize(2); + RR->topology->appendCertificateOfRepresentation(outp); + outp.setAt(corSizeAt,(uint16_t)((outp.size() - corSizeAt) - 2)); + RR->node->expectReplyTo(outp.packetId()); if (atAddress) { diff --git a/node/Peer.hpp b/node/Peer.hpp index bbe13a2e..e79739a3 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -439,7 +439,9 @@ private: } uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; - uint8_t _remoteClusterOptimal6[16]; + + const RuntimeEnvironment *RR; + uint64_t _lastReceive; // direct or indirect uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. uint64_t _lastTriedMemorizedPath; @@ -452,13 +454,17 @@ private: uint64_t _lastComRequestSent; uint64_t _lastCredentialsReceived; uint64_t _lastTrustEstablishedPacketReceived; - const RuntimeEnvironment *RR; + + uint8_t _remoteClusterOptimal6[16]; uint32_t _remoteClusterOptimal4; + uint16_t _vProto; uint16_t _vMajor; uint16_t _vMinor; uint16_t _vRevision; + Identity _id; + struct { uint64_t lastReceive; SharedPtr path; @@ -467,6 +473,7 @@ private: #endif } _paths[ZT_MAX_PEER_NETWORK_PATHS]; Mutex _paths_m; + unsigned int _numPaths; unsigned int _latency; unsigned int _directPathPushCutoffCount; diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 18916985..bc290e75 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -152,6 +152,8 @@ public: memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else throw std::runtime_error("invalid signature"); + } else { + p += 2 + b.template at(p); } p += 2 + b.template at(p); diff --git a/node/Tag.hpp b/node/Tag.hpp index 97228157..65348200 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -148,12 +148,14 @@ public: _issuedTo.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - if (b[p++] != 1) - throw std::runtime_error("unrecognized signature type"); - if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) - throw std::runtime_error("invalid signature length"); - p += 2; - memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + if (b[p++] == 1) { + if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) + throw std::runtime_error("invalid signature length"); + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + } else { + p += 2 + b.template at(p); + } p += 2 + b.template at(p); if (p > b.size()) diff --git a/node/Topology.cpp b/node/Topology.cpp index 0cd3db9e..f19d8656 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -417,6 +417,7 @@ void Topology::_memoizeUpstreams() // assumes _upstreams_m and _peers_m are locked _upstreamAddresses.clear(); _amRoot = false; + for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { if (i->identity == RR->identity) { _amRoot = true; @@ -429,6 +430,7 @@ void Topology::_memoizeUpstreams() } } } + for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { if (i->identity == RR->identity) { @@ -443,6 +445,15 @@ void Topology::_memoizeUpstreams() } } } + + std::sort(_upstreamAddresses.begin(),_upstreamAddresses.end()); + + _cor.clear(); + for(std::vector
::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { + if (!_cor.addRepresentative(*a)) + break; + } + _cor.sign(RR->identity,RR->node->now()); } } // namespace ZeroTier diff --git a/node/Topology.hpp b/node/Topology.hpp index 78dc0fe8..dca35789 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -38,6 +38,7 @@ #include "InetAddress.hpp" #include "Hashtable.hpp" #include "World.hpp" +#include "CertificateOfRepresentation.hpp" namespace ZeroTier { @@ -383,6 +384,25 @@ public: _trustedPathCount = count; } + /** + * @return Current certificate of representation (copy) + */ + inline CertificateOfRepresentation certificateOfRepresentation() const + { + Mutex::Lock _l(_upstreams_m); + return _cor; + } + + /** + * @param buf Buffer to receive COR + */ + template + void appendCertificateOfRepresentation(Buffer &buf) + { + Mutex::Lock _l(_upstreams_m); + _cor.serialize(buf); + } + private: Identity _getIdentity(const Address &zta); void _memoizeUpstreams(); @@ -404,6 +424,7 @@ private: std::vector _moons; std::vector
_contactingMoons; std::vector
_upstreamAddresses; + CertificateOfRepresentation _cor; bool _amRoot; Mutex _upstreams_m; // locks worlds, upstream info, moon info, etc. }; -- cgit v1.2.3 From 594cb1fad8db5b551982ae948d43a01b25494f05 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sat, 4 Feb 2017 19:29:39 -0800 Subject: Small fix for duplicates in world definitions. --- node/Topology.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index f19d8656..0fde63dc 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -421,7 +421,7 @@ void Topology::_memoizeUpstreams() for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { if (i->identity == RR->identity) { _amRoot = true; - } else { + } else if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),i->identity.address()) == _upstreamAddresses.end()) { _upstreamAddresses.push_back(i->identity.address()); SharedPtr &hp = _peers[i->identity.address()]; if (!hp) { @@ -435,7 +435,7 @@ void Topology::_memoizeUpstreams() for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { if (i->identity == RR->identity) { _amRoot = true; - } else { + } else if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),i->identity.address()) == _upstreamAddresses.end()) { _upstreamAddresses.push_back(i->identity.address()); SharedPtr &hp = _peers[i->identity.address()]; if (!hp) { -- cgit v1.2.3 From 43182f8f57483a47f1b44cdcf9dbb5387511afc2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Sun, 5 Feb 2017 16:19:03 -0800 Subject: Docs, code cleanup, and protect the extra new fields of HELLO with encryption as a precaution. --- doc/MANUAL.md | 12 +++---- node/Constants.hpp | 5 +++ node/Identity.cpp | 6 ++-- node/IncomingPacket.cpp | 88 +++++++++++++++++++++++++++++-------------------- node/Node.cpp | 4 +-- node/Packet.cpp | 62 +++++++++++++++++++++++----------- node/Packet.hpp | 31 ++++++++++++++--- node/Peer.cpp | 22 ++++++++----- node/Peer.hpp | 3 +- node/Salsa20.cpp | 4 +-- node/Salsa20.hpp | 34 +++---------------- node/Switch.cpp | 2 +- node/Utils.cpp | 4 +-- 13 files changed, 162 insertions(+), 115 deletions(-) (limited to 'node') diff --git a/doc/MANUAL.md b/doc/MANUAL.md index e84146f3..e0117a4d 100644 --- a/doc/MANUAL.md +++ b/doc/MANUAL.md @@ -72,13 +72,11 @@ This manual describes the design and operation of ZeroTier and its associated se ZeroTier is a smart Ethernet switch for planet Earth. -We've re-thought networking from first principles to deliver the flat end-to-end simplicity of the original pre-NAT pre-mobility Internet in a way that meets the security and mobility requirements of the 21st century. ZeroTier transforms the world into a unified modern data center where VPN, SDN, SD-WAN, and application peer to peer networking converge and where the distinction between the cloud and the endpoint largely disappears. All the complexity of managing these networking aspects as disparate systems is replaced by the simplicity of a single virtual cloud. +We've re-thought networking from first principles to deliver the flat end-to-end simplicity of the original pre-NAT pre-mobility Internet, yet in a way that meets the security and mobility needs of the 21st century. ZeroTier erases the distinction between cloud and endpoint, bringing modern data center SDN to every device regardless of its physical location. The objectives of VPN, SDN, SD-WAN, and application peer to peer networking can all be achieved together and with reduced complexity. -At first some users struggle with this paradigm, finding it difficult to forget the fragmentation and complexity that has accreted around networking over the past decade or two. We urge skeptical users to just try it and see how many networking acronyms vanish before their eyes. +Unlike most networking products it won't take you hours, days, or weeks to test or deploy ZeroTier. Most of the time everything just works with zero configuration, and most users with some level of TCP/IP knowledge can get up and running in minutes. More advanced features like federation, rules, micro-segmentation, capability based security credentials, network monitoring, and clustering are available but you don't need to worry about them until they're needed. -Unlike most networking products it won't take you hours, days, or weeks to test or deploy ZeroTier. Most of the time everything just works with zero configuration, and most users with some level of TCP/IP knowledge can get up and running in minutes. More advanced features like rules, micro-segmentation, capability based security credentials, network monitoring, and clustering are available but you don't need to worry about them until they're needed. - -The first section (2) of this guide explains ZeroTier's design and operation in detail and is written for users with at least an intermediate knowledge of topics like TCP/IP and Ethernet networking. Reading and understanding everything in it is not mandatory but we've written it as a deep technical dive as serious IT users typically like to understand the systems they deploy and use. Sections 3 and 4 deal more concretely with the ZeroTier One endpoint service software and how to deploy for common use cases. +The first section (2) of this guide explains ZeroTier's design and operation in detail and is written for users with at least an intermediate knowledge of topics like TCP/IP and Ethernet networking. All of it is not required reading for most users, but we've created a deep technical dive to satisfy the desire of IT professionals to understand the systems that they use. Sections 3 and 4 deal more concretely with the ZeroTier One endpoint service software and how to deploy for common use cases. ## **2.** How it Works @@ -88,7 +86,7 @@ ZeroTier is comprised of two closely coupled but conceptually distinct layers [i To build a planetary data center we first had to begin with the wiring. Tunneling into the Earth's core and putting a giant wire closet down there wasn't an option, so we decided to use software to build virtual wires over the existing Internet instead. -In conventional networks L1 (OSI layer 1) refers to the actual CAT5/CAT6 cables or wireless radio channels over which data is carried and the physical transciever chips that modulate and demodulate it. VL1 is a peer to peer network that does the same thing by using encryption, authentication, and a lot of networking tricks to create virtual wires as needed. +In conventional networks L1 (OSI layer 1) refers to the actual CAT5/CAT6 cables or wireless radio channels over which data is carried and the physical transciever chips that modulate and demodulate it. VL1 is a peer to peer network that does the same thing by using encryption, authentication, and a lot of networking tricks to create virtual wires on a dyniamic as-needed basis. ### **2.1.1.** Network Topology and Peer Discovery @@ -98,7 +96,7 @@ Roots run the same software as regular endpoints but reside at fast stable locat There is only one planet. Earth's root servers are operated by ZeroTier, Inc. as a free service. Their presence defines and unifies the global data center where we all reside. -Users can create "moons." These nominate additional roots for redundancy or performance. The most common reasons for doing this are to eliminate hard dependency on ZeroTier's third party infrastructure or to designate local roots inside your building or cloud so ZeroTier can work without a connection to the Internet. Moons are by no means required and most of our users get by just fine without them. +Moons can be created by users. These nominate additional roots for redundancy or performance. The most common reasons for doing this are to eliminate hard dependency on ZeroTier's third party infrastructure or to designate local roots inside your building or cloud so ZeroTier can work without a connection to the Internet. Moons are by no means required and most of our users get by just fine without them. When peers start out they have no direct links to one another, only upstream to roots. Every peer on VL1 possesses a globally unique address, but unlike IP addresses these are opaque cryptographic identifiers that encode no routing information. To communicate peers first send packets "up" the tree, and as these packets traverse the network they trigger the opportunistic creation of direct links along the way. The tree is constantly trying to "collapse itself" to optimize itself to the pattern of traffic it is carrying. diff --git a/node/Constants.hpp b/node/Constants.hpp index be4eb475..3bda3805 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -293,6 +293,11 @@ */ #define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) +/** + * Send a full HELLO every this often (ms) + */ +#define ZT_PEER_SEND_FULL_HELLO_EVERY (ZT_PEER_PING_PERIOD * 2) + /** * How often to retry expired paths that we're still remembering */ diff --git a/node/Identity.cpp b/node/Identity.cpp index 05b70873..89fdb836 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -46,7 +46,7 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub // but is not what we want for sequential memory-harndess. memset(genmem,0,ZT_IDENTITY_GEN_MEMORY); Salsa20 s20(digest,256,(char *)digest + 32); - s20.encrypt20((char *)genmem,(char *)genmem,64); + s20.crypt20((char *)genmem,(char *)genmem,64); for(unsigned long i=64;i(ZT_PROTO_VERB_HELLO_IDX_REVISION); const uint64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); - Identity id; - InetAddress externalSurfaceAddress; - uint64_t planetWorldId = 0; - uint64_t planetWorldTimestamp = 0; - std::vector< std::pair > moonIdsAndTimestamps; - { - unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); - - // Get external surface address if present (was not in old versions) - if (ptr < size()) - ptr += externalSurfaceAddress.deserialize(*this,ptr); - - // Get primary planet world ID and world timestamp if present - if ((ptr + 16) <= size()) { - planetWorldId = at(ptr); ptr += 8; - planetWorldTimestamp = at(ptr); - } - - // Get moon IDs and timestamps if present - if ((ptr + 2) <= size()) { - unsigned int numMoons = at(ptr); ptr += 2; - for(unsigned int i=0;i(at(ptr),at(ptr + 8))); - ptr += 16; - } - } + if (protoVersion < ZT_PROTO_VERSION_MIN) { + TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); + return true; } + Identity id; + unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); + if (fromAddress != id.address()) { - TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_path->address().toString().c_str()); - return true; - } - if (protoVersion < ZT_PROTO_VERSION_MIN) { - TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); + TRACE("dropped HELLO from %s(%s): identity does not match packet source address",fromAddress.toString().c_str(),_path->address().toString().c_str()); return true; } @@ -324,6 +299,43 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut // VALID -- if we made it here, packet passed identity and authenticity checks! + // Get external surface address if present (was not in old versions) + InetAddress externalSurfaceAddress; + if (ptr < size()) + ptr += externalSurfaceAddress.deserialize(*this,ptr); + + // Get primary planet world ID and world timestamp if present + uint64_t planetWorldId = 0; + uint64_t planetWorldTimestamp = 0; + if ((ptr + 16) <= size()) { + planetWorldId = at(ptr); ptr += 8; + planetWorldTimestamp = at(ptr); + } + + std::vector< std::pair > moonIdsAndTimestamps; + if (ptr < size()) { + // Remainder of packet, if present, is encrypted + cryptField(peer->key(),ptr,size() - ptr); + + // Get moon IDs and timestamps if present + if ((ptr + 2) <= size()) { + unsigned int numMoons = at(ptr); ptr += 2; + for(unsigned int i=0;i(at(ptr),at(ptr + 8))); + ptr += 16; + } + } + + // Handle COR if present (older versions don't send this) + if ((ptr + 2) <= size()) { + //const unsigned int corSize = at(ptr); ptr += 2; + ptr += 2; + CertificateOfRepresentation cor; + ptr += cor.deserialize(*this,ptr); + } + } + // Learn our external surface address from other peers to help us negotiate symmetric NATs // and detect changes to our global IP that can trigger path renegotiation. if ((externalSurfaceAddress)&&(hops() == 0)) @@ -337,6 +349,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR); outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR); outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); + if (protoVersion >= 5) { _path->address().serialize(outp); } else { @@ -387,6 +400,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut } outp.setAt(worldUpdateSizeAt,(uint16_t)(outp.size() - (worldUpdateSizeAt + 2))); + const unsigned int corSizeAt = outp.size(); + outp.addSize(2); + RR->topology->appendCertificateOfRepresentation(outp); + outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); + outp.armor(peer->key(),true); _path->send(RR,outp.data(),outp.size(),now); @@ -586,7 +604,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); if (RR->node->shouldUsePathForZeroTierTraffic(with,_path->localAddress(),atAddr)) { RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now()); + rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now(),false); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -1155,7 +1173,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now); + peer->attemptToContactAt(InetAddress(),a,now,false); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1174,7 +1192,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now); + peer->attemptToContactAt(InetAddress(),a,now,false); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } diff --git a/node/Node.cpp b/node/Node.cpp index 3d5b5c3d..b8e74a52 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -70,7 +70,7 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : Utils::getSecureRandom(foo,32); _prng.init(foo,256,foo); memset(_prngStream,0,sizeof(_prngStream)); - _prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream)); + _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); std::string idtmp(dataStoreGet("identity.secret")); if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { @@ -686,7 +686,7 @@ uint64_t Node::prng() { unsigned int p = (++_prngStreamPtr % ZT_NODE_PRNG_BUF_SIZE); if (!p) - _prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream)); + _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); return _prngStream[p]; } diff --git a/node/Packet.cpp b/node/Packet.cpp index 05fe8dd9..a1bb3132 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -18,9 +18,21 @@ #include #include +#include +#include +#include #include "Packet.hpp" +#ifdef _MSC_VER +#define FORCE_INLINE static __forceinline +#include +#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +#define FORCE_INLINE static inline +#endif + namespace ZeroTier { /************************************************************************** */ @@ -367,7 +379,7 @@ LZ4_decompress_*_continue() : #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include +//#include typedef struct { uint32_t hashTable[LZ4_HASH_SIZE_U32]; @@ -536,6 +548,7 @@ union LZ4_streamDecode_u { /*-************************************ * Compiler Options **************************************/ +#if 0 #ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline # include @@ -550,6 +563,7 @@ union LZ4_streamDecode_u { # define FORCE_INLINE static # endif #endif /* _MSC_VER */ +#endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) @@ -564,38 +578,39 @@ union LZ4_streamDecode_u { /*-************************************ * Memory routines **************************************/ -#include /* malloc, calloc, free */ +//#include /* malloc, calloc, free */ #define ALLOCATOR(n,s) calloc(n,s) #define FREEMEM free -#include /* memset, memcpy */ +//#include /* memset, memcpy */ #define MEM_INIT memset /*-************************************ * Basic Types **************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include +//#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +//# include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; typedef uint64_t U64; typedef uintptr_t uptrval; -#else +/*#else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; - typedef size_t uptrval; /* generally true, except OpenVMS-64 */ -#endif + typedef size_t uptrval; +#endif */ -#if defined(__x86_64__) - typedef U64 reg_t; /* 64-bits in x32 mode */ -#else - typedef size_t reg_t; /* 32-bits in x32 mode */ -#endif +typedef uintptr_t reg_t; +//#if defined(__x86_64__) +// typedef U64 reg_t; /* 64-bits in x32 mode */ +//#else +// typedef size_t reg_t; /* 32-bits in x32 mode */ +//#endif /*-************************************ * Reading and writing into memory @@ -606,7 +621,6 @@ static unsigned LZ4_isLittleEndian(void) return one.c[0]; } - #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) /* lie to the compiler about data alignment; use with caution */ @@ -1975,10 +1989,10 @@ void Packet::armor(const void *key,bool encryptPayload) // MAC key is always the first 32 bytes of the Salsa20 key stream // This is the same construction DJB's NaCl library uses - s20.encrypt12(ZERO_KEY,macKey,sizeof(macKey)); + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); if (encryptPayload) - s20.encrypt12(payload,payload,payloadLen); + s20.crypt12(payload,payload,payloadLen); Poly1305::compute(mac,payload,payloadLen,macKey); memcpy(field(ZT_PACKET_IDX_MAC,8),mac,8); @@ -1995,20 +2009,30 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); - Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)/*,ZT_PROTO_SALSA20_ROUNDS*/); + Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)); - s20.encrypt12(ZERO_KEY,macKey,sizeof(macKey)); + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); Poly1305::compute(mac,payload,payloadLen,macKey); if (!Utils::secureEq(mac,field(ZT_PACKET_IDX_MAC,8),8)) return false; if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) - s20.decrypt12(payload,payload,payloadLen); + s20.crypt12(payload,payload,payloadLen); return true; } else return false; // unrecognized cipher suite } +void Packet::cryptField(const void *key,unsigned int start,unsigned int len) +{ + unsigned char mangledKey[32]; + uint64_t iv = Utils::hton((uint64_t)start ^ at(ZT_PACKET_IDX_IV)); + _salsa20MangleKey((const unsigned char *)key,mangledKey); + Salsa20 s20(mangledKey,256,&iv); + unsigned char *const ptr = field(start,len); + s20.crypt12(ptr,ptr,len); +} + bool Packet::compress() { unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; diff --git a/node/Packet.hpp b/node/Packet.hpp index 7d404b25..03bd9ed3 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -542,6 +542,7 @@ public: * [<[...] destination address to which packet was sent>] * <[8] 64-bit world ID of current planet> * <[8] 64-bit timestamp of current planet> + * [... remainder if packet is encrypted using cryptField() ...] * <[2] 16-bit number of moons> * [<[1] 8-bit type ID of moon>] * [<[8] 64-bit world ID of moon>] @@ -550,9 +551,10 @@ public: * <[2] 16-bit length of certificate of representation> * [... certificate of representation ...] * - * HELLO is sent in the clear, and therefore cannot contain anything - * secret or highly confidential. It should contain nothing that is - * not either public or easy to obtain via other means. + * The initial fields of HELLO are sent in the clear. Fields after the + * planet definition (which are common knowledge) are however encrypted + * using the cryptField() function. The packet is MAC'd as usual using + * the same MAC construct as other packets. * * The destination address is the wire address to which this packet is * being sent, and in OK is *also* the destination address of the OK @@ -566,7 +568,7 @@ public: * 0x04 - 6-byte IPv4 UDP address/port -- format: <[4] IP>, <[2] port> * 0x06 - 18-byte IPv6 UDP address/port -- format: <[16] IP>, <[2] port> * - * OK payload: + * OK payload (note that OK is encrypted): * <[8] timestamp (echoed from original HELLO)> * <[1] protocol version (of responder)> * <[1] software major version (of responder)> @@ -576,6 +578,8 @@ public: * [<[...] destination address>] * <[2] 16-bit length of world update or 0 if none> * [[...] updates to planets and/or moons] + * <[2] 16-bit length of certificate of representation (of responder)> + * [... certificate of representation ...] * * ERROR has no payload. */ @@ -1348,6 +1352,25 @@ public: */ bool dearmor(const void *key); + /** + * Encrypt/decrypt a separately armored portion of a packet + * + * This keys using the same key in the same way as armor/dearmor, but + * uses a different IV computed from the packet's IV plus the starting + * point index. + * + * This currently uses Salsa20/12, but any message that uses this should + * incorporate a cipher selector to permit this to be changed later. + * + * This is currently only used to mask portions of HELLO as an extra + * security precation since most of that message is sent in the clear. + * + * @param key 32-byte key + * @param start Start of encrypted portion + * @param len Length of encrypted portion + */ + void cryptField(const void *key,unsigned int start,unsigned int len); + /** * Attempt to compress payload if not already (must be unencrypted) * diff --git a/node/Peer.cpp b/node/Peer.cpp index bb6b945d..338bea10 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -203,7 +203,7 @@ void Peer::received( #endif } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(path->localAddress(),path->address(),now); + attemptToContactAt(path->localAddress(),path->address(),now,true); path->sent(now); } } @@ -357,6 +357,8 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u outp.append((uint64_t)RR->topology->planetWorldId()); outp.append((uint64_t)RR->topology->planetWorldTimestamp()); + const unsigned int startCryptedPortionAt = outp.size(); + std::vector moons(RR->topology->moons()); outp.append((uint16_t)moons.size()); for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { @@ -368,21 +370,23 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u const unsigned int corSizeAt = outp.size(); outp.addSize(2); RR->topology->appendCertificateOfRepresentation(outp); - outp.setAt(corSizeAt,(uint16_t)((outp.size() - corSizeAt) - 2)); + outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); + + outp.cryptField(_key,startCryptedPortionAt,outp.size() - startCryptedPortionAt); RR->node->expectReplyTo(outp.packetId()); if (atAddress) { - outp.armor(_key,false); + outp.armor(_key,false); // false == don't encrypt full payload, but add MAC RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } else { - RR->sw->send(outp,false); + RR->sw->send(outp,false); // false == don't encrypt full payload, but add MAC } } -void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now) +void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello) { - if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) { + if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,true); @@ -398,7 +402,7 @@ void Peer::tryMemorizedPath(uint64_t now) _lastTriedMemorizedPath = now; InetAddress mp; if (RR->node->externalPathLookup(_id.address(),-1,mp)) - attemptToContactAt(InetAddress(),mp,now); + attemptToContactAt(InetAddress(),mp,now,true); } } @@ -420,7 +424,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) if (bestp >= 0) { if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { - attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now); + attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false); _paths[bestp].path->sent(now); } return true; @@ -444,7 +448,7 @@ void Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uin Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now); + attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now,false); _paths[p].path->sent(now); _paths[p].lastReceive = 0; // path will not be used unless it speaks again } diff --git a/node/Peer.hpp b/node/Peer.hpp index e79739a3..a3ec0088 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -161,8 +161,9 @@ public: * @param localAddr Local address * @param atAddress Destination address * @param now Current time + * @param sendFullHello If true, always send a full HELLO instead of just an ECHO */ - void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now); + void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello); /** * Try a memorized or statically defined path if any are known diff --git a/node/Salsa20.cpp b/node/Salsa20.cpp index 3aa19ac6..1a4641f7 100644 --- a/node/Salsa20.cpp +++ b/node/Salsa20.cpp @@ -123,7 +123,7 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) #endif } -void Salsa20::encrypt12(const void *in,void *out,unsigned int bytes) +void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) throw() { uint8_t tmp[64]; @@ -623,7 +623,7 @@ void Salsa20::encrypt12(const void *in,void *out,unsigned int bytes) } } -void Salsa20::encrypt20(const void *in,void *out,unsigned int bytes) +void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) throw() { uint8_t tmp[64]; diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 7e4c1e53..6405d450 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -56,51 +56,25 @@ public: throw(); /** - * Encrypt data using Salsa20/12 + * Encrypt/decrypt data using Salsa20/12 * * @param in Input data * @param out Output buffer * @param bytes Length of data */ - void encrypt12(const void *in,void *out,unsigned int bytes) + void crypt12(const void *in,void *out,unsigned int bytes) throw(); /** - * Encrypt data using Salsa20/20 + * Encrypt/decrypt data using Salsa20/20 * * @param in Input data * @param out Output buffer * @param bytes Length of data */ - void encrypt20(const void *in,void *out,unsigned int bytes) + void crypt20(const void *in,void *out,unsigned int bytes) throw(); - /** - * Decrypt data - * - * @param in Input data - * @param out Output buffer - * @param bytes Length of data - */ - inline void decrypt12(const void *in,void *out,unsigned int bytes) - throw() - { - encrypt12(in,out,bytes); - } - - /** - * Decrypt data - * - * @param in Input data - * @param out Output buffer - * @param bytes Length of data - */ - inline void decrypt20(const void *in,void *out,unsigned int bytes) - throw() - { - encrypt20(in,out,bytes); - } - private: union { #ifdef ZT_SALSA20_SSE diff --git a/node/Switch.cpp b/node/Switch.cpp index a769faea..346091a4 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -777,7 +777,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) if ((clusterMostRecentMemberId < 0)||(viaPath->lastIn() > clusterMostRecentTs)) { #endif if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now); + peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now,false); viaPath->sent(now); } #ifdef ZT_ENABLE_CLUSTER diff --git a/node/Utils.cpp b/node/Utils.cpp index 06b726cc..247dd54a 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -182,7 +182,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) #else // not __WINDOWS__ - static char randomBuf[131072]; + static char randomBuf[65536]; static unsigned int randomPtr = sizeof(randomBuf); static int devURandomFd = -1; @@ -215,7 +215,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) #endif // __WINDOWS__ or not - s20.encrypt12(buf,buf,bytes); + s20.crypt12(buf,buf,bytes); } bool Utils::scopy(char *dest,unsigned int len,const char *src) -- cgit v1.2.3 From f85a630a6438261fd21c149f32a77283e7bf76a3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 07:17:45 -0800 Subject: Docs and a small build fix in debug mode. --- node/IncomingPacket.cpp | 35 ++++++++++++++--------------- node/Packet.hpp | 58 ++++++++++++++++++++++++------------------------- 2 files changed, 46 insertions(+), 47 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 1bf70d68..49bcae11 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -212,15 +212,13 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION]; const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO_IDX_REVISION); const uint64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); + Identity id; + unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); if (protoVersion < ZT_PROTO_VERSION_MIN) { TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); return true; } - - Identity id; - unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); - if (fromAddress != id.address()) { TRACE("dropped HELLO from %s(%s): identity does not match packet source address",fromAddress.toString().c_str(),_path->address().toString().c_str()); return true; @@ -301,8 +299,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut // Get external surface address if present (was not in old versions) InetAddress externalSurfaceAddress; - if (ptr < size()) + if (ptr < size()) { ptr += externalSurfaceAddress.deserialize(*this,ptr); + if ((externalSurfaceAddress)&&(hops() == 0)) + RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + } // Get primary planet world ID and world timestamp if present uint64_t planetWorldId = 0; @@ -329,17 +330,16 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut // Handle COR if present (older versions don't send this) if ((ptr + 2) <= size()) { - //const unsigned int corSize = at(ptr); ptr += 2; - ptr += 2; - CertificateOfRepresentation cor; - ptr += cor.deserialize(*this,ptr); + if (at(ptr) > 0) { + CertificateOfRepresentation cor; + ptr += 2; + ptr += cor.deserialize(*this,ptr); + } else ptr += 2; } } - // Learn our external surface address from other peers to help us negotiate symmetric NATs - // and detect changes to our global IP that can trigger path renegotiation. - if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + // Send OK(HELLO) with an echo of the packet's timestamp and some of the same + // information about us: version, sent-to address, etc. Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_HELLO); @@ -466,10 +466,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p // Handle COR if present (older versions don't send this) if ((ptr + 2) <= size()) { - //const unsigned int corSize = at(ptr); ptr += 2; - ptr += 2; - CertificateOfRepresentation cor; - ptr += cor.deserialize(*this,ptr); + if (at(ptr) > 0) { + CertificateOfRepresentation cor; + ptr += 2; + ptr += cor.deserialize(*this,ptr); + } else ptr += 2; } TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); diff --git a/node/Packet.hpp b/node/Packet.hpp index 03bd9ed3..4859dafd 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -538,8 +538,7 @@ public: * <[2] software revision> * <[8] timestamp for determining latency> * <[...] binary serialized identity (see Identity)> - * <[1] destination address type> - * [<[...] destination address to which packet was sent>] + * <[...] physical destination address of packet> * <[8] 64-bit world ID of current planet> * <[8] 64-bit timestamp of current planet> * [... remainder if packet is encrypted using cryptField() ...] @@ -547,40 +546,39 @@ public: * [<[1] 8-bit type ID of moon>] * [<[8] 64-bit world ID of moon>] * [<[8] 64-bit timestamp of moon>] - * [... additional moons ...] + * [... additional moon type/ID/timestamp tuples ...] * <[2] 16-bit length of certificate of representation> * [... certificate of representation ...] * - * The initial fields of HELLO are sent in the clear. Fields after the - * planet definition (which are common knowledge) are however encrypted - * using the cryptField() function. The packet is MAC'd as usual using - * the same MAC construct as other packets. - * - * The destination address is the wire address to which this packet is - * being sent, and in OK is *also* the destination address of the OK - * packet. This can be used by the receiver to detect NAT, learn its real - * external address if behind NAT, and detect changes to its external - * address that require re-establishing connectivity. - * - * Destination address types and formats (not all of these are used now): - * 0x00 - None -- no destination address data present - * 0x01 - Ethernet address -- format: <[6] Ethernet MAC> - * 0x04 - 6-byte IPv4 UDP address/port -- format: <[4] IP>, <[2] port> - * 0x06 - 18-byte IPv6 UDP address/port -- format: <[16] IP>, <[2] port> - * - * OK payload (note that OK is encrypted): - * <[8] timestamp (echoed from original HELLO)> - * <[1] protocol version (of responder)> - * <[1] software major version (of responder)> - * <[1] software minor version (of responder)> - * <[2] software revision (of responder)> - * <[1] destination address type (for this OK, not copied from HELLO)> - * [<[...] destination address>] - * <[2] 16-bit length of world update or 0 if none> + * HELLO is sent in the clear as it is how peers share their identity + * public keys. A few additional fields are sent in the clear too, but + * these are things that are public info or are easy to determine. As + * of 1.2.0 we have added a few more fields, but since these could have + * the potential to be sensitive we introduced the encryption of the + * remainder of the packet. See cryptField(). Packet MAC is still + * performed of course, so authentication occurs as normal. + * + * Destination address is the actual wire address to which the packet + * was sent. See InetAddress::serialize() for format. + * + * OK payload: + * <[8] HELLO timestamp field echo> + * <[1] protocol version> + * <[1] software major version> + * <[1] software minor version> + * <[2] software revision> + * <[...] physical destination address of packet> + * <[2] 16-bit length of world update(s) or 0 if none> * [[...] updates to planets and/or moons] - * <[2] 16-bit length of certificate of representation (of responder)> + * <[2] 16-bit length of certificate of representation> * [... certificate of representation ...] * + * With the exception of the timestamp, the other fields pertain to the + * respondent who is sending OK and are not echoes. + * + * Note that OK is fully encrypted so no selective cryptField() of + * potentially sensitive fields is needed. + * * ERROR has no payload. */ VERB_HELLO = 0x01, -- cgit v1.2.3 From 803f74634a7e88e4e2781e4b8df7afeda3d968be Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 07:39:38 -0800 Subject: Tweak how we do crypto of the masked portions of HELLO just to be more "boring" in the DJB sense. --- node/Packet.cpp | 6 ++++-- node/Packet.hpp | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index a1bb3132..e685bcaa 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -2026,9 +2026,11 @@ bool Packet::dearmor(const void *key) void Packet::cryptField(const void *key,unsigned int start,unsigned int len) { unsigned char mangledKey[32]; - uint64_t iv = Utils::hton((uint64_t)start ^ at(ZT_PACKET_IDX_IV)); + unsigned char macKey[32]; _salsa20MangleKey((const unsigned char *)key,mangledKey); - Salsa20 s20(mangledKey,256,&iv); + mangledKey[0] ^= 1; // slightly alter key for this use case as an added guard against key stream reuse + Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)); + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); // discard the first 32 bytes of key stream (the ones use for MAC in armor()) as a precaution unsigned char *const ptr = field(start,len); s20.crypt12(ptr,ptr,len); } diff --git a/node/Packet.hpp b/node/Packet.hpp index 4859dafd..b736b84a 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1353,16 +1353,18 @@ public: /** * Encrypt/decrypt a separately armored portion of a packet * - * This keys using the same key in the same way as armor/dearmor, but - * uses a different IV computed from the packet's IV plus the starting - * point index. - * * This currently uses Salsa20/12, but any message that uses this should - * incorporate a cipher selector to permit this to be changed later. + * incorporate a cipher selector to permit this to be changed later. To + * ensure that key stream is not reused, the key is slightly altered for + * this use case and the same initial 32 keystream bytes that are taken + * for MAC in ordinary armor() are also skipped here. * * This is currently only used to mask portions of HELLO as an extra * security precation since most of that message is sent in the clear. * + * This must NEVER be used more than once in the same packet, as doing + * so will result in re-use of the same key stream. + * * @param key 32-byte key * @param start Start of encrypted portion * @param len Length of encrypted portion -- cgit v1.2.3 From e0d63c50db120df9fa6fa750d12c8fb2e621cae7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 07:45:57 -0800 Subject: One more tweak after thinking about related keys and key stream reuse. Just a precaution. --- node/Packet.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index e685bcaa..790f4b09 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -2028,7 +2028,9 @@ void Packet::cryptField(const void *key,unsigned int start,unsigned int len) unsigned char mangledKey[32]; unsigned char macKey[32]; _salsa20MangleKey((const unsigned char *)key,mangledKey); - mangledKey[0] ^= 1; // slightly alter key for this use case as an added guard against key stream reuse + mangledKey[0] ^= 0x7f; + mangledKey[1] ^= ((start >> 8) & 0xff); + mangledKey[2] ^= (start & 0xff); // slightly alter key for this use case as an added guard against key stream reuse Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)); s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); // discard the first 32 bytes of key stream (the ones use for MAC in armor()) as a precaution unsigned char *const ptr = field(start,len); -- cgit v1.2.3 From 21f4a97c35c01200040deeb3051cebc923072ac5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 11:49:41 -0800 Subject: CSPRNG performance improvement, self test build fix. --- node/Utils.cpp | 39 ++++++++++++++++++++++----------------- selftest.cpp | 12 ++++++------ 2 files changed, 28 insertions(+), 23 deletions(-) (limited to 'node') diff --git a/node/Utils.cpp b/node/Utils.cpp index 247dd54a..00b0db06 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -144,6 +144,8 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) static Mutex globalLock; static Salsa20 s20; static bool s20Initialized = false; + static uint8_t randomBuf[65536]; + static unsigned int randomPtr = sizeof(randomBuf); Mutex::Lock _l(globalLock); @@ -168,27 +170,31 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) static HCRYPTPROV cryptProvider = NULL; - if (cryptProvider == NULL) { - if (!CryptAcquireContextA(&cryptProvider,NULL,NULL,PROV_RSA_FULL,CRYPT_VERIFYCONTEXT|CRYPT_SILENT)) { - fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to obtain WinCrypt context!\r\n"); - exit(1); - return; + for(unsigned int i=0;i= sizeof(randomBuf)) { + if (cryptProvider == NULL) { + if (!CryptAcquireContextA(&cryptProvider,NULL,NULL,PROV_RSA_FULL,CRYPT_VERIFYCONTEXT|CRYPT_SILENT)) { + fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to obtain WinCrypt context!\r\n"); + exit(1); + } + } + if (!CryptGenRandom(cryptProvider,(DWORD)sizeof(randomBuf),(BYTE *)randomBuf)) { + fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() CryptGenRandom failed!\r\n"); + exit(1); + } + randomPtr = 0; + s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); } - } - if (!CryptGenRandom(cryptProvider,(DWORD)bytes,(BYTE *)buf)) { - fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() CryptGenRandom failed!\r\n"); - exit(1); + ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } #else // not __WINDOWS__ - static char randomBuf[65536]; - static unsigned int randomPtr = sizeof(randomBuf); static int devURandomFd = -1; - if (devURandomFd <= 0) { + if (devURandomFd < 0) { devURandomFd = ::open("/dev/urandom",O_RDONLY); - if (devURandomFd <= 0) { + if (devURandomFd < 0) { fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to open /dev/urandom\n"); exit(1); return; @@ -201,7 +207,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) if ((int)::read(devURandomFd,randomBuf,sizeof(randomBuf)) != (int)sizeof(randomBuf)) { ::close(devURandomFd); devURandomFd = ::open("/dev/urandom",O_RDONLY); - if (devURandomFd <= 0) { + if (devURandomFd < 0) { fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to open /dev/urandom\n"); exit(1); return; @@ -209,13 +215,12 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } else break; } randomPtr = 0; + s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); } - ((char *)buf)[i] = randomBuf[randomPtr++]; + ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } #endif // __WINDOWS__ or not - - s20.crypt12(buf,buf,bytes); } bool Utils::scopy(char *dest,unsigned int len,const char *src) diff --git a/selftest.cpp b/selftest.cpp index 6054983d..b1f4b54a 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -154,9 +154,9 @@ static int testCrypto() memset(buf3,0,sizeof(buf3)); Salsa20 s20; s20.init("12345678123456781234567812345678",256,"12345678"); - s20.encrypt20(buf1,buf2,sizeof(buf1)); + s20.crypt20(buf1,buf2,sizeof(buf1)); s20.init("12345678123456781234567812345678",256,"12345678"); - s20.decrypt20(buf2,buf3,sizeof(buf2)); + s20.crypt20(buf2,buf3,sizeof(buf2)); if (memcmp(buf1,buf3,sizeof(buf1))) { std::cout << "FAIL (encrypt/decrypt test)" << std::endl; return -1; @@ -165,7 +165,7 @@ static int testCrypto() Salsa20 s20(s20TV0Key,256,s20TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); - s20.encrypt20(buf1,buf2,64); + s20.crypt20(buf1,buf2,64); if (memcmp(buf2,s20TV0Ks,64)) { std::cout << "FAIL (test vector 0)" << std::endl; return -1; @@ -173,7 +173,7 @@ static int testCrypto() s20.init(s2012TV0Key,256,s2012TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); - s20.encrypt12(buf1,buf2,64); + s20.crypt12(buf1,buf2,64); if (memcmp(buf2,s2012TV0Ks,64)) { std::cout << "FAIL (test vector 1)" << std::endl; return -1; @@ -195,7 +195,7 @@ static int testCrypto() double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { - s20.encrypt12(bb,bb,1234567); + s20.crypt12(bb,bb,1234567); bytes += 1234567.0; } uint64_t end = OSUtils::now(); @@ -213,7 +213,7 @@ static int testCrypto() double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { - s20.encrypt20(bb,bb,1234567); + s20.crypt20(bb,bb,1234567); bytes += 1234567.0; } uint64_t end = OSUtils::now(); -- cgit v1.2.3 From 435e4c4695024702a8493e66d802652f116741f8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 12:06:10 -0800 Subject: Fix HELLO parse bug. --- node/IncomingPacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 49bcae11..ddf93244 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -310,7 +310,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut uint64_t planetWorldTimestamp = 0; if ((ptr + 16) <= size()) { planetWorldId = at(ptr); ptr += 8; - planetWorldTimestamp = at(ptr); + planetWorldTimestamp = at(ptr); ptr += 8; } std::vector< std::pair > moonIdsAndTimestamps; -- cgit v1.2.3 From 9ddc2a4331e9dfa9d5aecd1c782d31527cf6e572 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 14:00:49 -0800 Subject: Add a break action to rules engine to make capabilities easier to use. --- include/ZeroTierOne.h | 4 ++-- node/Network.cpp | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 6c50a0a6..860343ba 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -563,9 +563,9 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_ACTION_REDIRECT = 4, /** - * Log if match and if rule debugging is enabled in the build, otherwise does nothing (for developers) + * Stop evaluating rule set (drops unless there are capabilities, etc.) */ - ZT_NETWORK_RULE_ACTION_DEBUG_LOG = 5, + ZT_NETWORK_RULE_ACTION_BREAK = 5, /** * Maximum ID for an ACTION, anything higher is a MATCH diff --git a/node/Network.cpp b/node/Network.cpp index c5855418..77810964 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -53,7 +53,7 @@ static const char *_rtn(const ZT_VirtualNetworkRuleType rt) case ZT_NETWORK_RULE_ACTION_TEE: return "ACTION_TEE"; case ZT_NETWORK_RULE_ACTION_WATCH: return "ACTION_WATCH"; case ZT_NETWORK_RULE_ACTION_REDIRECT: return "ACTION_REDIRECT"; - case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: return "ACTION_DEBUG_LOG"; + case ZT_NETWORK_RULE_ACTION_BREAK: return "ACTION_BREAK"; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: return "MATCH_SOURCE_ZEROTIER_ADDRESS"; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: return "MATCH_DEST_ZEROTIER_ADDRESS"; case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; @@ -251,13 +251,12 @@ static _doZtFilterResult _doZtFilter( } } continue; - // This is a no-op that exists for use with rules engine tracing and isn't for use in production - case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: // a no-op target specifically for debugging purposes + case ZT_NETWORK_RULE_ACTION_BREAK: #ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_DEBUG_LOG",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); + _dumpFilterTrace("ACTION_BREAK",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); dlog.clear(); #endif // ZT_RULES_ENGINE_DEBUGGING - continue; + return DOZTFILTER_NO_MATCH; // Unrecognized ACTIONs are ignored as no-ops default: -- cgit v1.2.3 From 78d548458be2dd9b7520e95a285d94a6a9f86c75 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 16:38:48 -0800 Subject: Capabilities basically work but need to refactor a bit for performance reasons. --- node/Membership.cpp | 16 +++++----------- node/Membership.hpp | 3 --- node/Network.cpp | 2 +- 3 files changed, 6 insertions(+), 15 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index d7c7c0e6..f847b465 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -32,7 +32,6 @@ namespace ZeroTier { Membership::Membership() : _lastUpdatedMulticast(0), - _lastPushAttempt(0), _lastPushedCom(0), _comRevocationThreshold(0) { @@ -42,12 +41,7 @@ Membership::Membership() : void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) { - // This limits how often we go through this logic, which prevents us from - // doing all this for every single packet or other event. - if ( ((now - _lastPushAttempt) < 1000ULL) && (!force) ) - return; - _lastPushAttempt = now; - + //TRACE("pushCredentials() to %s localCapabilityIndex==%d force==%d",peerAddress.toString().c_str(),localCapabilityIndex,(int)force); try { unsigned int localTagPtr = 0; bool needCom = ( (nconf.com) && ( ((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); @@ -182,21 +176,21 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCapability *)0; if (have) { if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->cap.timestamp() > cap.timestamp()) ) { - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(Capability) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; } if (have->cap == cap) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_ACCEPTED_REDUNDANT; } } switch(cap.verify(RR)) { default: - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(Capability) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; case 0: - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); if (!have) have = _newCapability(cap.id()); have->lastReceived = RR->node->now(); have->cap = cap; diff --git a/node/Membership.hpp b/node/Membership.hpp index c54aec9b..9814dce8 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -266,9 +266,6 @@ private: // Last time we pushed MULTICAST_LIKE(s) uint64_t _lastUpdatedMulticast; - // Last time we checked if credential push was needed - uint64_t _lastPushAttempt; - // Last time we pushed our COM to this peer uint64_t _lastPushedCom; diff --git a/node/Network.cpp b/node/Network.cpp index 77810964..5961b087 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1150,8 +1150,8 @@ bool Network::gate(const SharedPtr &peer) if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { if (!m) m = &(_membership(peer->address())); - m->pushCredentials(RR,now,peer->address(),_config,-1,false); if (m->shouldLikeMulticasts(now)) { + m->pushCredentials(RR,now,peer->address(),_config,-1,false); _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); m->likingMulticasts(now); } -- cgit v1.2.3 From 59ba7c8bf5788edf248771f24cd4e7e14a67e162 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 17:10:20 -0800 Subject: Improve efficiency of pushCredentials() method since it gets called a lot. --- node/Membership.cpp | 96 +++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 51 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index f847b465..5bb23a1a 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -41,59 +41,53 @@ Membership::Membership() : void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) { - //TRACE("pushCredentials() to %s localCapabilityIndex==%d force==%d",peerAddress.toString().c_str(),localCapabilityIndex,(int)force); - try { - unsigned int localTagPtr = 0; - bool needCom = ( (nconf.com) && ( ((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); - do { - Buffer capsAndTags; - - unsigned int appendedCaps = 0; - if (localCapabilityIndex >= 0) { - capsAndTags.addSize(2); - - if ( (_localCaps[localCapabilityIndex].id != nconf.capabilities[localCapabilityIndex].id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCaps[localCapabilityIndex].lastPushed = now; - _localCaps[localCapabilityIndex].id = nconf.capabilities[localCapabilityIndex].id(); - nconf.capabilities[localCapabilityIndex].serialize(capsAndTags); - ++appendedCaps; - } - - capsAndTags.setAt(0,(uint16_t)appendedCaps); - localCapabilityIndex = -1; // don't send this cap again on subsequent loops if force is true - } else { - capsAndTags.append((uint16_t)0); - } + bool sendCom = ( (nconf.com) && ( ((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); - unsigned int appendedTags = 0; - const unsigned int tagCountPos = capsAndTags.size(); - capsAndTags.addSize(2); - for(;localTagPtr= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - if ((capsAndTags.size() + sizeof(Tag)) >= (ZT_PROTO_MAX_PACKET_LENGTH - sizeof(CertificateOfMembership))) - break; - nconf.tags[localTagPtr].serialize(capsAndTags); - ++appendedTags; - } - } - capsAndTags.setAt(tagCountPos,(uint16_t)appendedTags); - - if (needCom||appendedCaps||appendedTags) { - Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); - if (needCom) { - nconf.com.serialize(outp); - _lastPushedCom = now; - } - outp.append((uint8_t)0x00); - outp.append(capsAndTags.data(),capsAndTags.size()); - outp.append((uint16_t)0); // no revocations, these propagate differently - outp.compress(); - RR->sw->send(outp,true); - needCom = false; // don't send COM again on subsequent loops if force is true + const Capability *sendCap; + if (localCapabilityIndex >= 0) { + sendCap = &(nconf.capabilities[localCapabilityIndex]); + if ( (_localCaps[localCapabilityIndex].id != sendCap->id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCaps[localCapabilityIndex].lastPushed = now; + _localCaps[localCapabilityIndex].id = sendCap->id(); + } else sendCap = (const Capability *)0; + } else sendCap = (const Capability *)0; + + unsigned int tagPtr = 0; + while ((tagPtr < nconf.tagCount)||(sendCom)||(sendCap)) { + Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + + if (sendCom) { + sendCom = false; + nconf.com.serialize(outp); + _lastPushedCom = now; + } + outp.append((uint8_t)0x00); + + if (sendCap) { + outp.append((uint16_t)1); + sendCap->serialize(outp); + sendCap = (const Capability *)0; + } else outp.append((uint16_t)0); + + const unsigned int tagCountAt = outp.size(); + outp.addSize(2); + unsigned int thisPacketTagCount = 0; + while ((tagPtr < nconf.tagCount)&&((outp.size() + sizeof(Tag) + 32) < ZT_PROTO_MAX_PACKET_LENGTH)) { + if ( (_localTags[tagPtr].id != nconf.tags[tagPtr].id()) || ((now - _localTags[tagPtr].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localTags[tagPtr].lastPushed = now; + _localTags[tagPtr].id = nconf.tags[tagPtr].id(); + nconf.tags[tagPtr].serialize(outp); + ++thisPacketTagCount; } - } while (localTagPtr < nconf.tagCount); - } catch ( ... ) { - TRACE("unable to send credentials due to unexpected exception"); + ++tagPtr; + } + outp.setAt(tagCountAt,(uint16_t)thisPacketTagCount); + + // No revocations, these propagate differently + outp.append((uint16_t)0); + + outp.compress(); + RR->sw->send(outp,true); } } -- cgit v1.2.3 From 723a9a6e9aa4254c7d740f9af6596ba8450924ac Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Feb 2017 17:20:22 -0800 Subject: Small additional efficiency improvement. --- node/Membership.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index 5bb23a1a..6307b85d 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -52,8 +52,18 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now } else sendCap = (const Capability *)0; } else sendCap = (const Capability *)0; + const Tag *sendTags[ZT_MAX_NETWORK_TAGS]; + unsigned int sendTagCount = 0; + for(unsigned int t=0;t= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localTags[t].lastPushed = now; + _localTags[t].id = nconf.tags[t].id(); + sendTags[sendTagCount++] = &(nconf.tags[t]); + } + } + unsigned int tagPtr = 0; - while ((tagPtr < nconf.tagCount)||(sendCom)||(sendCap)) { + while ((tagPtr < sendTagCount)||(sendCom)||(sendCap)) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (sendCom) { @@ -72,11 +82,9 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now const unsigned int tagCountAt = outp.size(); outp.addSize(2); unsigned int thisPacketTagCount = 0; - while ((tagPtr < nconf.tagCount)&&((outp.size() + sizeof(Tag) + 32) < ZT_PROTO_MAX_PACKET_LENGTH)) { + while ((tagPtr < sendTagCount)&&((outp.size() + sizeof(Tag) + 32) < ZT_PROTO_MAX_PACKET_LENGTH)) { if ( (_localTags[tagPtr].id != nconf.tags[tagPtr].id()) || ((now - _localTags[tagPtr].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localTags[tagPtr].lastPushed = now; - _localTags[tagPtr].id = nconf.tags[tagPtr].id(); - nconf.tags[tagPtr].serialize(outp); + sendTags[tagPtr]->serialize(outp); ++thisPacketTagCount; } ++tagPtr; -- cgit v1.2.3 From 672f17c6e9ae981a70c51fdd62882e4847c5b6ba Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 7 Feb 2017 09:33:39 -0800 Subject: Add a mask and value range to the IP tos rule field. This allows TOS to be matched more usefully. This will break anyone using tos in the beta, but nobody seems to be and its pre-release so now is the time. --- controller/EmbeddedNetworkController.cpp | 8 ++++++-- include/ZeroTierOne.h | 5 ++++- node/Capability.hpp | 10 +++++++--- node/Network.cpp | 8 +++++--- 4 files changed, 22 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 90018f0d..e798a80c 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -150,7 +150,9 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; - r["ipTos"] = (unsigned int)rule.v.ipTos; + r["mask"] = (unsigned int)rule.v.ipTos.mask; + r["start"] = (unsigned int)rule.v.ipTos.value[0]; + r["end"] = (unsigned int)rule.v.ipTos.value[1]; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: r["type"] = "MATCH_IP_PROTOCOL"; @@ -329,7 +331,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_IP_TOS") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(OSUtils::jsonInt(r["ipTos"],0ULL) & 0xffULL); + rule.v.ipTos.mask = (uint8_t)(OSUtils::jsonInt(r["mask"],0ULL) & 0xffULL); + rule.v.ipTos.value[0] = (uint8_t)(OSUtils::jsonInt(r["start"],0ULL) & 0xffULL); + rule.v.ipTos.value[1] = (uint8_t)(OSUtils::jsonInt(r["end"],0ULL) & 0xffULL); return true; } else if (t == "MATCH_IP_PROTOCOL") { rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 860343ba..583f9b6a 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -705,7 +705,10 @@ typedef struct /** * IP type of service a.k.a. DSCP field */ - uint8_t ipTos; + struct { + uint8_t mask; + uint8_t value[2]; + } ipTos; /** * Ethernet packet size in host byte order (start-end, inclusive) diff --git a/node/Capability.hpp b/node/Capability.hpp index ddbfd9ee..08714038 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -216,8 +216,10 @@ public: b.append((uint8_t)rules[i].v.ipv6.mask); break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - b.append((uint8_t)1); - b.append((uint8_t)rules[i].v.ipTos); + b.append((uint8_t)3); + b.append((uint8_t)rules[i].v.ipTos.mask); + b.append((uint8_t)rules[i].v.ipTos.value[0]); + b.append((uint8_t)rules[i].v.ipTos.value[1]); break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: b.append((uint8_t)1); @@ -308,7 +310,9 @@ public: rules[ruleCount].v.ipv6.mask = (uint8_t)b[p + 16]; break; case ZT_NETWORK_RULE_MATCH_IP_TOS: - rules[ruleCount].v.ipTos = (uint8_t)b[p]; + rules[ruleCount].v.ipTos.mask = (uint8_t)b[p]; + rules[ruleCount].v.ipTos.value[0] = (uint8_t)b[p+1]; + rules[ruleCount].v.ipTos.value[1] = (uint8_t)b[p+2]; break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: rules[ruleCount].v.ipProtocol = (uint8_t)b[p]; diff --git a/node/Network.cpp b/node/Network.cpp index 5961b087..7412e3e7 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -368,11 +368,13 @@ static _doZtFilterResult _doZtFilter( break; case ZT_NETWORK_RULE_MATCH_IP_TOS: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + //thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); + const uint8_t tosMasked = frameData[1] & rules[rn].v.ipTos.mask; + thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((frameData[1] & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { - const uint8_t trafficClass = ((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f); - thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((trafficClass & 0xfc) >> 2)); + const uint8_t tosMasked = (((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f)) & rules[rn].v.ipTos.mask; + thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((trafficClass & 0xfc) >> 2),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; -- cgit v1.2.3 From cdc289fa9c5d7d19990c14946ede5f3642e235d2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 7 Feb 2017 14:06:40 -0800 Subject: Tags work. --- node/IncomingPacket.cpp | 2 +- node/Membership.cpp | 7 ++----- node/Network.cpp | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ddf93244..02d6a140 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -78,7 +78,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } if (!uncompress()) { - TRACE("dropped packet from %s(%s), compressed data invalid",sourceAddress.toString().c_str(),_path->address().toString().c_str()); + TRACE("dropped packet from %s(%s), compressed data invalid (verb may be %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),(unsigned int)verb()); return true; } diff --git a/node/Membership.cpp b/node/Membership.cpp index 6307b85d..8c6dab64 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -83,11 +83,8 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now outp.addSize(2); unsigned int thisPacketTagCount = 0; while ((tagPtr < sendTagCount)&&((outp.size() + sizeof(Tag) + 32) < ZT_PROTO_MAX_PACKET_LENGTH)) { - if ( (_localTags[tagPtr].id != nconf.tags[tagPtr].id()) || ((now - _localTags[tagPtr].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - sendTags[tagPtr]->serialize(outp); - ++thisPacketTagCount; - } - ++tagPtr; + sendTags[tagPtr++]->serialize(outp); + ++thisPacketTagCount; } outp.setAt(tagCountAt,(uint16_t)thisPacketTagCount); diff --git a/node/Network.cpp b/node/Network.cpp index 7412e3e7..461e1c20 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -371,11 +371,11 @@ static _doZtFilterResult _doZtFilter( //thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); const uint8_t tosMasked = frameData[1] & rules[rn].v.ipTos.mask; thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); - FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((frameData[1] & 0xfc) >> 2),(unsigned int)thisRuleMatches); + FILTER_TRACE("%u %s %c (IPv4) %u&%u==%u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)tosMasked,(unsigned int)rules[rn].v.ipTos.mask,(unsigned int)rules[rn].v.ipTos.value[0],(unsigned int)rules[rn].v.ipTos.value[1],(unsigned int)thisRuleMatches); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { const uint8_t tosMasked = (((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f)) & rules[rn].v.ipTos.mask; thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); - FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipTos,(unsigned int)((trafficClass & 0xfc) >> 2),(unsigned int)thisRuleMatches); + FILTER_TRACE("%u %s %c (IPv4) %u&%u==%u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)tosMasked,(unsigned int)rules[rn].v.ipTos.mask,(unsigned int)rules[rn].v.ipTos.value[0],(unsigned int)rules[rn].v.ipTos.value[1],(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); -- cgit v1.2.3 From 42f28bce52c3342e3aac68488260a02c71691177 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Feb 2017 09:03:48 -0800 Subject: Cleanup and make moons (federated roots) a little easier to deal with. --- include/ZeroTierOne.h | 9 +++-- node/IncomingPacket.cpp | 1 + node/Node.cpp | 39 ++++++++++--------- node/Node.hpp | 2 +- node/Topology.cpp | 99 +++++++++++++++++++++++++++++------------------- node/Topology.hpp | 27 +++++-------- node/Utils.cpp | 19 +++------- node/Utils.hpp | 3 +- service/ControlPlane.cpp | 4 +- service/OneService.cpp | 2 +- 10 files changed, 107 insertions(+), 98 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 583f9b6a..38ae7d8a 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -835,9 +835,9 @@ enum ZT_VirtualNetworkConfigOperation */ enum ZT_PeerRole { - ZT_PEER_ROLE_LEAF = 0, // ordinary node - ZT_PEER_ROLE_UPSTREAM = 1, // moon root - ZT_PEER_ROLE_ROOT = 2 // planetary root + ZT_PEER_ROLE_LEAF = 0, // ordinary node + ZT_PEER_ROLE_MOON = 1, // moon root + ZT_PEER_ROLE_PLANET = 2 // planetary root }; /** @@ -1790,10 +1790,11 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint * called for each on startup. * * @param moonWorldId Moon's world ID + * @param moonSeed If non-zero, the ZeroTier address of any member of the moon to query for moon definition * @param len Length of moonWorld in bytes * @return Error if moon was invalid or failed to be added */ -enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId); +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed); /** * Remove a moon (does nothing if not present) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 02d6a140..8836df9f 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -568,6 +568,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr id.serialize(outp,false); ++count; } else { + // Request unknown WHOIS from upstream from us (if we have one) RR->sw->requestWhois(addr); #ifdef ZT_ENABLE_CLUSTER // Distribute WHOIS queries across a cluster if we do not know the ID. diff --git a/node/Node.cpp b/node/Node.cpp index b8e74a52..388a4fb2 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -156,23 +156,26 @@ ZT_ResultCode Node::processVirtualNetworkFrame( class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), + _upstreamsToContact(upstreamsToContact), _now(now), _bestCurrentUpstream(RR->topology->getUpstreamPeer()) { - RR->topology->getUpstreamStableEndpoints(_upstreams); } uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay inline void operator()(Topology &t,const SharedPtr &p) { - const std::vector *const upstreamStableEndpoints = _upstreams.get(p->address()); + const std::vector *const upstreamStableEndpoints = _upstreamsToContact.get(p->address()); if (upstreamStableEndpoints) { bool contacted = false; + // Upstreams must be pinged constantly over both IPv4 and IPv6 to allow + // them to perform three way handshake introductions for both stacks. + if (!p->doPingAndKeepalive(_now,AF_INET)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; @@ -183,7 +186,6 @@ public: } } } else contacted = true; - if (!p->doPingAndKeepalive(_now,AF_INET6)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; @@ -202,6 +204,7 @@ public: } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); + _upstreamsToContact.erase(p->address()); // erase from upstreams to contact so that we can WHOIS those that remain } else if (p->isActive(_now)) { p->doPingAndKeepalive(_now,-1); } @@ -209,9 +212,9 @@ public: private: const RuntimeEnvironment *RR; + Hashtable< Address,std::vector > &_upstreamsToContact; const uint64_t _now; const SharedPtr _bestCurrentUpstream; - Hashtable< Address,std::vector > _upstreams; }; ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) @@ -238,17 +241,19 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) (*n)->requestConfiguration(); - // Attempt to get identity for any unknown upstreams - const std::vector
upstreams(RR->topology->upstreamAddresses()); - for(std::vector
::const_iterator a(upstreams.begin());a!=upstreams.end();++a) { - if (!RR->topology->getPeer(*a)) - RR->sw->requestWhois(*a); - } - // Do pings and keepalives - _PingPeersThatNeedPing pfunc(RR,now); + Hashtable< Address,std::vector > upstreamsToContact; + RR->topology->getUpstreamsToContact(upstreamsToContact); + _PingPeersThatNeedPing pfunc(RR,upstreamsToContact,now); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); + // Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds) + Hashtable< Address,std::vector >::Iterator i(upstreamsToContact); + Address *upstreamAddress = (Address *)0; + std::vector *upstreamStableEndpoints = (std::vector *)0; + while (i.next(upstreamAddress,upstreamStableEndpoints)) + RR->sw->requestWhois(*upstreamAddress); + // Update online status, post status change as event const bool oldOnline = _online; _online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot())); @@ -337,9 +342,9 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } -ZT_ResultCode Node::orbit(uint64_t moonWorldId) +ZT_ResultCode Node::orbit(uint64_t moonWorldId,uint64_t moonSeed) { - RR->topology->addMoon(moonWorldId); + RR->topology->addMoon(moonWorldId,Address(moonSeed)); return ZT_RESULT_OK; } @@ -919,10 +924,10 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint } } -enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId) +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed) { try { - return reinterpret_cast(node)->orbit(moonWorldId); + return reinterpret_cast(node)->orbit(moonWorldId,moonSeed); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } diff --git a/node/Node.hpp b/node/Node.hpp index 3e742092..d83ce968 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -95,7 +95,7 @@ public: ZT_ResultCode leave(uint64_t nwid,void **uptr); ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); - ZT_ResultCode orbit(uint64_t moonWorldId); + ZT_ResultCode orbit(uint64_t moonWorldId,uint64_t moonSeed); ZT_ResultCode deorbit(uint64_t moonWorldId); uint64_t address() const; void status(ZT_NodeStatus *status) const; diff --git a/node/Topology.cpp b/node/Topology.cpp index 0fde63dc..d85b6a7d 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -27,19 +27,31 @@ namespace ZeroTier { -// 2015-11-16 -- The Fabulous Four (should have named them after Beatles!) -//#define ZT_DEFAULT_WORLD_LENGTH 494 -//static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x51,0x11,0x70,0xb2,0xfb,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0x80,0x31,0xa4,0x65,0x95,0x45,0x06,0x1c,0xfb,0xc2,0x4e,0x5d,0xe7,0x0a,0x40,0x7a,0x97,0xce,0x36,0xa2,0x3d,0x05,0xca,0x87,0xc7,0x59,0x27,0x5c,0x8b,0x0d,0x4c,0xb4,0xbb,0x26,0x2f,0x77,0x17,0x5e,0xb7,0x4d,0xb8,0xd3,0xb4,0xe9,0x23,0x5d,0xcc,0xa2,0x71,0xa8,0xdf,0xf1,0x23,0xa3,0xb2,0x66,0x74,0xea,0xe5,0xdc,0x8d,0xef,0xd3,0x0a,0xa9,0xac,0xcb,0xda,0x93,0xbd,0x6c,0xcd,0x43,0x1d,0xa7,0x98,0x6a,0xde,0x70,0xc0,0xc6,0x1c,0xaf,0xf0,0xfd,0x7f,0x8a,0xb9,0x76,0x13,0xe1,0xde,0x4f,0xf3,0xd6,0x13,0x04,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x01,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x01,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09}; - -// 2015-11-20 -- Alice and Bob are live, and we're now IPv6 dual-stack! -//#define ZT_DEFAULT_WORLD_LENGTH 792 -//static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x51,0x26,0x6f,0x7c,0x8a,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0xe8,0x0a,0xf5,0xbc,0xf8,0x3d,0x97,0xcd,0xc3,0xf8,0xe2,0x41,0x16,0x42,0x0f,0xc7,0x76,0x8e,0x07,0xf3,0x7e,0x9e,0x7d,0x1b,0xb3,0x23,0x21,0x79,0xce,0xb9,0xd0,0xcb,0xb5,0x94,0x7b,0x89,0x21,0x57,0x72,0xf6,0x70,0xa1,0xdd,0x67,0x38,0xcf,0x45,0x45,0xc2,0x8d,0x46,0xec,0x00,0x2c,0xe0,0x2a,0x63,0x3f,0x63,0x8d,0x33,0x08,0x51,0x07,0x77,0x81,0x5b,0x32,0x49,0xae,0x87,0x89,0xcf,0x31,0xaa,0x41,0xf1,0x52,0x97,0xdc,0xa2,0x55,0xe1,0x4a,0x6e,0x3c,0x04,0xf0,0x4f,0x8a,0x0e,0xe9,0xca,0xec,0x24,0x30,0x04,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x33,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x02,0x00,0xe0,0x01,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0xb7,0x40,0x01,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x6a,0x30,0x01,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x64,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09}; - -// 2015-12-17 -- Old New York root is dead, old SF still alive -//#define ZT_DEFAULT_WORLD_LENGTH 732 -//static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x51,0xb1,0x7e,0x39,0x9d,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0x8a,0xca,0xf2,0x3d,0x71,0x2e,0xc2,0x39,0x45,0x66,0xb3,0xe9,0x39,0x79,0xb1,0x55,0xc4,0xa9,0xfc,0xbc,0xfc,0x55,0xaf,0x8a,0x2f,0x38,0xc8,0xcd,0xe9,0x02,0x5b,0x86,0xa9,0x72,0xf7,0x16,0x00,0x35,0xb7,0x84,0xc9,0xfc,0xe4,0xfa,0x96,0x8b,0xf4,0x1e,0xba,0x60,0x9f,0x85,0x14,0xc2,0x07,0x4b,0xfd,0xd1,0x6c,0x19,0x69,0xd3,0xf9,0x09,0x9c,0x9d,0xe3,0xb9,0x8f,0x11,0x78,0x71,0xa7,0x4a,0x05,0xd8,0xcc,0x60,0xa2,0x06,0x66,0x9f,0x47,0xc2,0x71,0xb8,0x54,0x80,0x9c,0x45,0x16,0x10,0xa9,0xd0,0xbd,0xf7,0x03,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x33,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x02,0x00,0xe0,0x01,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0xb7,0x40,0x01,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x6a,0x30,0x01,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x64,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x02,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0xc5,0xf0,0x01,0x27,0x09}; - -// 2016-01-13 -- Old San Francisco 1.0.1 root is dead, now we're just on Alice and Bob! +/* + * 2016-01-13 ZeroTier planet definition for the third planet of Sol: + * + * There are two roots, each of which is a cluster spread across multiple + * continents and providers. They are named Alice and Bob after the + * canonical example names used in cryptography. + * + * Alice: + * + * root-alice-ams-01: Amsterdam, Netherlands + * root-alice-joh-01: Johannesburg, South Africa + * root-alice-nyc-01: New York, New York, USA + * root-alice-sao-01: Sao Paolo, Brazil + * root-alice-sfo-01: San Francisco, California, USA + * root-alice-sgp-01: Singapore + * + * Bob: + * + * root-bob-dfw-01: Dallas, Texas, USA + * root-bob-fra-01: Frankfurt, Germany + * root-bob-par-01: Paris, France + * root-bob-syd-01: Sydney, Australia + * root-bob-tok-01: Tokyo, Japan + * root-bob-tor-01: Toronto, Canada + */ #define ZT_DEFAULT_WORLD_LENGTH 634 static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x52,0x3c,0x32,0x50,0x1a,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0x4a,0xf7,0x86,0xa8,0x40,0xd6,0x52,0xea,0xae,0x9e,0x7a,0xbf,0x4c,0x97,0x66,0xab,0x2d,0x6f,0xaf,0xc9,0x2b,0x3a,0xff,0xed,0xd6,0x30,0x3e,0xc4,0x6a,0x65,0xf2,0xbd,0x83,0x52,0xf5,0x40,0xe9,0xcc,0x0d,0x6e,0x89,0x3f,0x9a,0xa0,0xb8,0xdf,0x42,0xd2,0x2f,0x84,0xe6,0x03,0x26,0x0f,0xa8,0xe3,0xcc,0x05,0x05,0x03,0xef,0x12,0x80,0x0d,0xce,0x3e,0xb6,0x58,0x3b,0x1f,0xa8,0xad,0xc7,0x25,0xf9,0x43,0x71,0xa7,0x5c,0x9a,0xc7,0xe1,0xa3,0xb8,0x88,0xd0,0x71,0x6c,0x94,0x99,0x73,0x41,0x0b,0x1b,0x48,0x84,0x02,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x33,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x02,0x00,0xe0,0x01,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0xb7,0x40,0x01,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x6a,0x30,0x01,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x64,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09}; @@ -219,9 +231,9 @@ ZT_PeerRole Topology::role(const Address &ztaddr) const if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) { for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { if (i->identity.address() == ztaddr) - return ZT_PEER_ROLE_ROOT; + return ZT_PEER_ROLE_PLANET; } - return ZT_PEER_ROLE_UPSTREAM; + return ZT_PEER_ROLE_MOON; } return ZT_PEER_ROLE_LEAF; } @@ -290,22 +302,32 @@ bool Topology::addWorld(const World &newWorld) if (existing->shouldBeReplacedBy(newWorld)) *existing = newWorld; else return false; - } else if ((newWorld.type() == World::TYPE_MOON)&&(std::find(_contactingMoons.begin(),_contactingMoons.end(),Address(newWorld.id() >> 24)) != _contactingMoons.end())) { - _moons.push_back(newWorld); - existing = &(_moons.back()); - - std::vector
cm; - for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { - if (m->toInt() != ((existing->id() >> 24) & 0xffffffffffULL)) - cm.push_back(*m); + } else if (newWorld.type() == World::TYPE_MOON) { + for(std::vector< std::pair >::const_iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) { + if (m->first == newWorld.id()) { + for(std::vector::const_iterator r(newWorld.roots().begin());r!=newWorld.roots().end();++r) { + if (r->identity.address() == m->second) { + _moonSeeds.erase(m); + m = _moonSeeds.end(); // cause outer loop to terminate + _moons.push_back(newWorld); + existing = &(_moons.back()); + break; + } + } + } } - _contactingMoons.swap(cm); - } else return false; + if (!existing) + return false; + } else { + return false; + } char savePath[64]; - if (existing->type() == World::TYPE_MOON) + if (existing->type() == World::TYPE_MOON) { Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",existing->id()); - else Utils::scopy(savePath,sizeof(savePath),"planet"); + } else { + Utils::scopy(savePath,sizeof(savePath),"planet"); + } try { Buffer dswtmp; existing->serialize(dswtmp,false); @@ -319,15 +341,8 @@ bool Topology::addWorld(const World &newWorld) return true; } -void Topology::addMoon(const uint64_t id) +void Topology::addMoon(const uint64_t id,const Address &seed) { - { - const Address a(id >> 24); - Mutex::Lock _l(_upstreams_m); - if (std::find(_contactingMoons.begin(),_contactingMoons.end(),a) == _contactingMoons.end()) - _contactingMoons.push_back(a); - } - char savePath[64]; Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); @@ -344,7 +359,11 @@ void Topology::addMoon(const uint64_t id) } } catch ( ... ) {} - RR->node->dataStorePut(savePath,"\0",1,false); // persist that we want to be a member + if (seed) { + Mutex::Lock _l(_upstreams_m); + if (std::find(_moonSeeds.begin(),_moonSeeds.end(),std::pair(id,seed)) == _moonSeeds.end()) + _moonSeeds.push_back(std::pair(id,seed)); + } } void Topology::removeMoon(const uint64_t id) @@ -364,12 +383,12 @@ void Topology::removeMoon(const uint64_t id) } _moons.swap(nm); - std::vector
cm; - for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { - if (m->toInt() != ((id >> 24) & 0xffffffffffULL)) + std::vector< std::pair > cm; + for(std::vector< std::pair >::const_iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) { + if (m->first != id) cm.push_back(*m); } - _contactingMoons.swap(cm); + _moonSeeds.swap(cm); _memoizeUpstreams(); } diff --git a/node/Topology.hpp b/node/Topology.hpp index dca35789..2465de64 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -170,14 +170,11 @@ public: bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const; /** - * This gets the known stable endpoints for any upstream - * - * It also adds empty entries for any upstreams we are attempting to - * contact. + * Gets upstreams to contact and their stable endpoints (if known) * * @param eps Hash table to fill with addresses and their stable endpoints */ - inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector > &eps) const + inline void getUpstreamsToContact(Hashtable< Address,std::vector > &eps) const { Mutex::Lock _l(_upstreams_m); for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { @@ -196,8 +193,8 @@ public: } } } - for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) - eps[*m]; + for(std::vector< std::pair >::const_iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) + eps[m->second]; } /** @@ -206,12 +203,7 @@ public: inline std::vector
upstreamAddresses() const { Mutex::Lock _l(_upstreams_m); - std::vector
u(_upstreamAddresses); - for(std::vector
::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) { - if (std::find(u.begin(),u.end(),*m) == u.end()) - u.push_back(*m); - } - return u; + return _upstreamAddresses; } /** @@ -260,13 +252,12 @@ public: * Add a moon * * This loads it from moons.d if present, and if not adds it to - * a list of moons that we want to contact. It does not actually - * send anything, though this will happen on the next background - * task loop where pings etc. are checked. + * a list of moons that we want to contact. * * @param id Moon ID + * @param seed If non-NULL, an address of any member of the moon to contact */ - void addMoon(const uint64_t id); + void addMoon(const uint64_t id,const Address &seed); /** * Remove a moon @@ -422,7 +413,7 @@ private: World _planet; std::vector _moons; - std::vector
_contactingMoons; + std::vector< std::pair > _moonSeeds; std::vector
_upstreamAddresses; CertificateOfRepresentation _cor; bool _amRoot; diff --git a/node/Utils.cpp b/node/Utils.cpp index 00b0db06..fb448dd6 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -47,21 +47,14 @@ namespace ZeroTier { const char Utils::HEXCHARS[16] = { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; -static void _Utils_doBurn(char *ptr,unsigned int len) +// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. +static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) { - for(unsigned int i=0;irole) { case ZT_PEER_ROLE_LEAF: prole = "LEAF"; break; - case ZT_PEER_ROLE_UPSTREAM: prole = "UPSTREAM"; break; - case ZT_PEER_ROLE_ROOT: prole = "ROOT"; break; + case ZT_PEER_ROLE_MOON: prole = "MOON"; break; + case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break; } Utils::snprintf(json,sizeof(json), diff --git a/service/OneService.cpp b/service/OneService.cpp index 9a1503e5..6465463b 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -707,7 +707,7 @@ public: for(std::vector::iterator f(moonsDotD.begin());f!=moonsDotD.end();++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16)&&(f->substr(16) == ".moon")) - _node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str())); + _node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str()),0); } } -- cgit v1.2.3 From e6840a1863a2ed996d7fe66321d753e003d00375 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Feb 2017 09:26:05 -0800 Subject: Can't erase from vector using const_iterator on some C++ compilers.' --- node/Topology.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index d85b6a7d..8d0ed929 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -303,7 +303,7 @@ bool Topology::addWorld(const World &newWorld) *existing = newWorld; else return false; } else if (newWorld.type() == World::TYPE_MOON) { - for(std::vector< std::pair >::const_iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) { + for(std::vector< std::pair >::iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) { if (m->first == newWorld.id()) { for(std::vector::const_iterator r(newWorld.roots().begin());r!=newWorld.roots().end();++r) { if (r->identity.address() == m->second) { -- cgit v1.2.3 From e4b6611201bb2f69c05a4c104c77d6ec51c2c38b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Feb 2017 09:46:34 -0800 Subject: Only accept world updates from upstreams. --- node/IncomingPacket.cpp | 20 ++++++++++++-------- node/Peer.cpp | 8 +++++++- node/Topology.hpp | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 8836df9f..c6cf7f36 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -449,22 +449,26 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p InetAddress externalSurfaceAddress; unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2; - // Get reported external surface address if present (was not on old versions) + // Get reported external surface address if present if (ptr < size()) ptr += externalSurfaceAddress.deserialize(*this,ptr); - // Handle planet or moon updates if present (older versions don't send this) + // Handle planet or moon updates if present if ((ptr + 2) <= size()) { const unsigned int worldLen = at(ptr); ptr += 2; - const unsigned int endOfWorlds = ptr + worldLen; - while (ptr < endOfWorlds) { - World w; - ptr += w.deserialize(*this,ptr); - RR->topology->addWorld(w); + if (RR->topology->isUpstream(peer->identity())) { + const unsigned int endOfWorlds = ptr + worldLen; + while (ptr < endOfWorlds) { + World w; + ptr += w.deserialize(*this,ptr); + RR->topology->addWorld(w); + } + } else { + ptr += worldLen; } } - // Handle COR if present (older versions don't send this) + // Handle certificate of representation if present if ((ptr + 2) <= size()) { if (at(ptr) > 0) { CertificateOfRepresentation cor; diff --git a/node/Peer.cpp b/node/Peer.cpp index 338bea10..d5847092 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -360,12 +360,18 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u const unsigned int startCryptedPortionAt = outp.size(); std::vector moons(RR->topology->moons()); - outp.append((uint16_t)moons.size()); + std::vector moonsWanted(RR->topology->moonsWanted()); + outp.append((uint16_t)(moons.size() + moonsWanted.size())); for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { outp.append((uint8_t)m->type()); outp.append((uint64_t)m->id()); outp.append((uint64_t)m->timestamp()); } + for(std::vector::const_iterator m(moonsWanted.begin());m!=moonsWanted.end();++m) { + outp.append((uint8_t)World::TYPE_MOON); + outp.append(*m); + outp.append((uint64_t)0); + } const unsigned int corSizeAt = outp.size(); outp.addSize(2); diff --git a/node/Topology.hpp b/node/Topology.hpp index 2465de64..35f98ccc 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -215,6 +215,20 @@ public: return _moons; } + /** + * @return Moon IDs we are waiting for from seeds + */ + inline std::vector moonsWanted() const + { + Mutex::Lock _l(_upstreams_m); + std::vector mw; + for(std::vector< std::pair >::const_iterator s(_moonSeeds.begin());s!=_moonSeeds.end();++s) { + if (std::find(mw.begin(),mw.end(),s->first) == mw.end()) + mw.push_back(s->first); + } + return mw; + } + /** * @return Current planet */ -- cgit v1.2.3 From 4b115665057a334c08e427e0f34a4e3a48514ea1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Feb 2017 14:27:08 -0800 Subject: Integrate moon concept into http config bus, and clean up that code quite a bit. --- include/ZeroTierOne.h | 10 - node/Node.cpp | 12 +- node/Node.hpp | 6 + node/World.hpp | 10 + service/ControlPlane.cpp | 577 +++++++++++++++++++---------------------------- service/ControlPlane.hpp | 4 +- service/OneService.cpp | 2 +- 7 files changed, 263 insertions(+), 358 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 38ae7d8a..9690489a 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -442,16 +442,6 @@ typedef struct */ uint64_t address; - /** - * Current world ID - */ - uint64_t worldId; - - /** - * Current world revision/timestamp - */ - uint64_t worldTimestamp; - /** * Public identity in string-serialized form (safe to send to others) * diff --git a/node/Node.cpp b/node/Node.cpp index 388a4fb2..6dc89387 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -362,8 +362,6 @@ uint64_t Node::address() const void Node::status(ZT_NodeStatus *status) const { status->address = RR->identity.address().toInt(); - status->worldId = RR->topology->planetWorldId(); - status->worldTimestamp = RR->topology->planetWorldTimestamp(); status->publicIdentity = RR->publicIdentityStr.c_str(); status->secretIdentity = RR->secretIdentityStr.c_str(); status->online = _online ? 1 : 0; @@ -714,6 +712,16 @@ void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_ RR->topology->setTrustedPaths(reinterpret_cast(networks),ids,count); } +World Node::planet() const +{ + return RR->topology->planet(); +} + +std::vector Node::moons() const +{ + return RR->topology->moons(); +} + void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) { if (destination == RR->identity.address()) { diff --git a/node/Node.hpp b/node/Node.hpp index d83ce968..a1d4b719 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -24,6 +24,7 @@ #include #include +#include #include "Constants.hpp" @@ -54,6 +55,8 @@ namespace ZeroTier { +class World; + /** * Implementation of Node object as defined in CAPI * @@ -210,6 +213,9 @@ public: void postCircuitTestReport(const ZT_CircuitTestReport *report); void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); + World planet() const; + std::vector moons() const; + /** * Register that we are expecting a reply to a packet ID * diff --git a/node/World.hpp b/node/World.hpp index 8fe6dd2e..6e835bec 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -135,6 +135,16 @@ public: */ inline uint64_t timestamp() const { return _ts; } + /** + * @return C25519 signature + */ + inline const C25519::Signature &signature() const { return _signature; } + + /** + * @return Public key that must sign next update + */ + inline const C25519::Public &updatesMustBeSignedBy() const { return _updatesMustBeSignedBy; } + /** * Check whether a world update should replace this one * diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 0b0fbc09..100deeda 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -35,77 +35,17 @@ #include "../node/InetAddress.hpp" #include "../node/Node.hpp" #include "../node/Utils.hpp" +#include "../node/World.hpp" + #include "../osdep/OSUtils.hpp" namespace ZeroTier { -static std::string _jsonEscape(const char *s) -{ - std::string buf; - for(const char *p=s;(*p);++p) { - switch(*p) { - case '\t': buf.append("\\t"); break; - case '\b': buf.append("\\b"); break; - case '\r': buf.append("\\r"); break; - case '\n': buf.append("\\n"); break; - case '\f': buf.append("\\f"); break; - case '"': buf.append("\\\""); break; - case '\\': buf.append("\\\\"); break; - case '/': buf.append("\\/"); break; - default: buf.push_back(*p); break; - } - } - return buf; -} -static std::string _jsonEscape(const std::string &s) { return _jsonEscape(s.c_str()); } +namespace { -static std::string _jsonEnumerate(const struct sockaddr_storage *ss,unsigned int count) +static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc,const std::string &portDeviceName,const OneService::NetworkSettings &localSettings) { - std::string buf; - buf.push_back('['); - for(unsigned int i=0;i 0) - buf.push_back(','); - buf.push_back('"'); - buf.append(_jsonEscape(reinterpret_cast(&(ss[i]))->toString())); - buf.push_back('"'); - } - buf.push_back(']'); - return buf; -} -static std::string _jsonEnumerate(const ZT_VirtualNetworkRoute *routes,unsigned int count) -{ - std::string buf; - buf.push_back('['); - for(unsigned int i=0;i 0) - buf.push_back(','); - buf.append("{\"target\":\""); - buf.append(_jsonEscape(reinterpret_cast(&(routes[i].target))->toString())); - buf.append("\",\"via\":"); - if (routes[i].via.ss_family == routes[i].target.ss_family) { - buf.push_back('"'); - buf.append(_jsonEscape(reinterpret_cast(&(routes[i].via))->toIpString())); - buf.append("\","); - } else buf.append("null,"); - char tmp[1024]; - Utils::snprintf(tmp,sizeof(tmp),"\"flags\":%u,\"metric\":%u}",(unsigned int)routes[i].flags,(unsigned int)routes[i].metric); - buf.append(tmp); - } - buf.push_back(']'); - return buf; -} - -static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_VirtualNetworkConfig *nc,const std::string &portDeviceName,const OneService::NetworkSettings &localSettings) -{ - char json[4096]; - char prefix[32]; - - if (depth >= sizeof(prefix)) // sanity check -- shouldn't be possible - return; - for(unsigned int i=0;istatus) { @@ -121,107 +61,48 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_VirtualNetw case ZT_NETWORK_TYPE_PUBLIC: ntype = "PUBLIC"; break; } - std::string allowManaged = (localSettings.allowManaged) ? "true" : "false"; - if (localSettings.allowManagedWhitelist.size() != 0) { - allowManaged = ""; - for (InetAddress address : localSettings.allowManagedWhitelist) { - if (allowManaged.size() != 0) allowManaged += ','; - allowManaged += address.toIpString() + "/" + std::to_string(address.netmaskBits()); - } + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); + nj["id"] = tmp; + nj["nwid"] = tmp; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); + nj["mac"] = tmp; + nj["name"] = nc->name; + nj["status"] = nstatus; + nj["type"] = ntype; + nj["mtu"] = nc->mtu; + nj["dhcp"] = (bool)(nc->dhcp == 0); + nj["bridge"] = (bool)(nc->bridge == 0); + nj["broadcastEnabled"] = (bool)(nc->broadcastEnabled == 0); + nj["portError"] = nc->portError; + nj["netconfRevision"] = nc->netconfRevision; + nj["portDeviceName"] = portDeviceName; + nj["allowManaged"] = localSettings.allowManaged; + nj["allowGlobal"] = localSettings.allowGlobal; + nj["allowDefault"] = localSettings.allowDefault; + + nlohmann::json aa = nlohmann::json::array(); + for(unsigned int i=0;iassignedAddressCount;++i) { + aa.push_back(reinterpret_cast(&(nc->assignedAddresses[i]))->toString()); } - - Utils::snprintf(json,sizeof(json), - "%s{\n" - "%s\t\"id\": \"%.16llx\",\n" - "%s\t\"nwid\": \"%.16llx\",\n" - "%s\t\"mac\": \"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x\",\n" - "%s\t\"name\": \"%s\",\n" - "%s\t\"status\": \"%s\",\n" - "%s\t\"type\": \"%s\",\n" - "%s\t\"mtu\": %u,\n" - "%s\t\"dhcp\": %s,\n" - "%s\t\"bridge\": %s,\n" - "%s\t\"broadcastEnabled\": %s,\n" - "%s\t\"portError\": %d,\n" - "%s\t\"netconfRevision\": %lu,\n" - "%s\t\"assignedAddresses\": %s,\n" - "%s\t\"routes\": %s,\n" - "%s\t\"portDeviceName\": \"%s\",\n" - "%s\t\"allowManaged\": %s,\n" - "%s\t\"allowGlobal\": %s,\n" - "%s\t\"allowDefault\": %s\n" - "%s}", - prefix, - prefix,nc->nwid, - prefix,nc->nwid, - prefix,(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff), - prefix,_jsonEscape(nc->name).c_str(), - prefix,nstatus, - prefix,ntype, - prefix,nc->mtu, - prefix,(nc->dhcp == 0) ? "false" : "true", - prefix,(nc->bridge == 0) ? "false" : "true", - prefix,(nc->broadcastEnabled == 0) ? "false" : "true", - prefix,nc->portError, - prefix,nc->netconfRevision, - prefix,_jsonEnumerate(nc->assignedAddresses,nc->assignedAddressCount).c_str(), - prefix,_jsonEnumerate(nc->routes,nc->routeCount).c_str(), - prefix,_jsonEscape(portDeviceName).c_str(), - prefix,allowManaged.c_str(), - prefix,(localSettings.allowGlobal) ? "true" : "false", - prefix,(localSettings.allowDefault) ? "true" : "false", - prefix); - buf.append(json); -} - -static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath *pp,unsigned int count) -{ - char json[2048]; - char prefix[32]; - - if (depth >= sizeof(prefix)) // sanity check -- shouldn't be possible - return std::string(); - for(unsigned int i=0;i 0) - buf.push_back(','); - Utils::snprintf(json,sizeof(json), - "{\n" - "%s\t\"address\": \"%s\",\n" - "%s\t\"lastSend\": %llu,\n" - "%s\t\"lastReceive\": %llu,\n" - "%s\t\"active\": %s,\n" - "%s\t\"expired\": %s,\n" - "%s\t\"preferred\": %s,\n" - "%s\t\"trustedPathId\": %llu\n" - "%s}", - prefix,_jsonEscape(reinterpret_cast(&(pp[i].address))->toString()).c_str(), - prefix,pp[i].lastSend, - prefix,pp[i].lastReceive, - prefix,(pp[i].expired != 0) ? "false" : "true", - prefix,(pp[i].expired == 0) ? "false" : "true", - prefix,(pp[i].preferred == 0) ? "false" : "true", - prefix,pp[i].trustedPathId, - prefix); - buf.append(json); + nj["assignedAddresses"] = aa; + + nlohmann::json ra = nlohmann::json::array(); + for(unsigned int i=0;irouteCount;++i) { + nlohmann::json rj; + rj["target"] = reinterpret_cast(&(nc->routes[i].target))->toString(); + if (nc->routes[i].via.ss_family == nc->routes[i].target.ss_family) + rj["via"] = reinterpret_cast(&(nc->routes[i].via))->toIpString(); + else rj["via"] = nlohmann::json(); + rj["flags"] = (int)nc->routes[i].flags; + rj["metric"] = (int)nc->routes[i].metric; + ra.push_back(rj); } - return buf; + nj["routes"] = ra; } -static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) +static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) { - char json[2048]; - char prefix[32]; - - if (depth >= sizeof(prefix)) // sanity check -- shouldn't be possible - return; - for(unsigned int i=0;irole) { @@ -230,39 +111,57 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break; } - Utils::snprintf(json,sizeof(json), - "%s{\n" - "%s\t\"address\": \"%.10llx\",\n" - "%s\t\"versionMajor\": %d,\n" - "%s\t\"versionMinor\": %d,\n" - "%s\t\"versionRev\": %d,\n" - "%s\t\"version\": \"%d.%d.%d\",\n" - "%s\t\"latency\": %u,\n" - "%s\t\"role\": \"%s\",\n" - "%s\t\"paths\": [%s]\n" - "%s}", - prefix, - prefix,peer->address, - prefix,peer->versionMajor, - prefix,peer->versionMinor, - prefix,peer->versionRev, - prefix,peer->versionMajor,peer->versionMinor,peer->versionRev, - prefix,peer->latency, - prefix,prole, - prefix,_jsonEnumerate(depth+1,peer->paths,peer->pathCount).c_str(), - prefix); - buf.append(json); + Utils::snprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + pj["address"] = tmp; + pj["versionMajor"] = peer->versionMajor; + pj["versionMinor"] = peer->versionMinor; + pj["versionRev"] = peer->versionRev; + Utils::snprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); + pj["version"] = tmp; + pj["latency"] = peer->latency; + pj["role"] = prole; + + nlohmann::json pa = nlohmann::json::array(); + for(unsigned int i=0;ipathCount;++i) { + nlohmann::json j; + j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(); + j["lastSend"] = peer->paths[i].lastSend; + j["lastReceive"] = peer->paths[i].lastReceive; + j["active"] = (bool)(peer->paths[i].expired != 0); + j["expired"] = (bool)(peer->paths[i].expired == 0); + j["preferred"] = (bool)(peer->paths[i].preferred == 0); + j["trustedPathId"] = peer->paths[i].trustedPathId; + pa.push_back(j); + } + pj["paths"] = pa; } -ControlPlane::ControlPlane(OneService *svc,Node *n,const char *uiStaticPath) : - _svc(svc), - _node(n), - _controller((EmbeddedNetworkController *)0), - _uiStaticPath((uiStaticPath) ? uiStaticPath : "") +static void _moonToJson(nlohmann::json &mj,const World &world) { + mj["id"] = world.id(); + mj["timestamp"] = world.timestamp(); + mj["signature"] = Utils::hex(world.signature().data,world.signature().size()); + mj["updatesMustBeSignedBy"] = Utils::hex(world.updatesMustBeSignedBy().data,world.updatesMustBeSignedBy().size()); + nlohmann::json ra = nlohmann::json::array(); + for(std::vector::const_iterator r(world.roots().begin());r!=world.roots().end();++r) { + nlohmann::json rj; + rj["identity"] = r->identity.toString(false); + nlohmann::json eps = nlohmann::json::array(); + for(std::vector::const_iterator a(r->stableEndpoints.begin());a!=r->stableEndpoints.end();++a) + eps.push_back(a->toString()); + rj["stableEndpoints"] = eps; + ra.push_back(rj); + } + mj["roots"] = ra; + mj["active"] = true; } -ControlPlane::~ControlPlane() +} // anonymous namespace + +ControlPlane::ControlPlane(OneService *svc,Node *n) : + _svc(svc), + _node(n), + _controller((EmbeddedNetworkController *)0) { } @@ -275,10 +174,12 @@ unsigned int ControlPlane::handleRequest( std::string &responseBody, std::string &responseContentType) { - char json[8194]; + char tmp[256]; unsigned int scode = 404; + nlohmann::json res; std::vector ps(OSUtils::split(path.c_str(),"/","","")); std::map urlArgs; + Mutex::Lock _l(_lock); /* Note: this is kind of restricted in what it'll take. It does not support @@ -298,8 +199,6 @@ unsigned int ControlPlane::handleRequest( else urlArgs[a->substr(0,eqpos)] = a->substr(eqpos + 1); } } - } else { - ps.push_back(std::string("index.html")); } bool isAuth = false; @@ -315,148 +214,113 @@ unsigned int ControlPlane::handleRequest( } if (httpMethod == HTTP_GET) { - - std::string ext; - std::size_t dotIdx = ps[0].find_last_of('.'); - if (dotIdx != std::string::npos) - ext = ps[0].substr(dotIdx); - - if ((ps.size() == 1)&&(ext.length() >= 2)&&(ext[0] == '.')) { - /* Static web pages can be served without authentication to enable a simple web - * UI. This is still only allowed from approved IP addresses. Anything with a - * dot in the first path element (e.g. foo.html) is considered a static page, - * as nothing in the API is so named. */ - - if (_uiStaticPath.length() > 0) { - if (ext == ".html") - responseContentType = "text/html"; - else if (ext == ".js") - responseContentType = "application/javascript"; - else if (ext == ".jsx") - responseContentType = "text/jsx"; - else if (ext == ".json") - responseContentType = "application/json"; - else if (ext == ".css") - responseContentType = "text/css"; - else if (ext == ".png") - responseContentType = "image/png"; - else if (ext == ".jpg") - responseContentType = "image/jpeg"; - else if (ext == ".gif") - responseContentType = "image/gif"; - else if (ext == ".txt") - responseContentType = "text/plain"; - else if (ext == ".xml") - responseContentType = "text/xml"; - else if (ext == ".svg") - responseContentType = "image/svg+xml"; - else responseContentType = "application/octet-stream"; - scode = OSUtils::readFile((_uiStaticPath + ZT_PATH_SEPARATOR_S + ps[0]).c_str(),responseBody) ? 200 : 404; - } else { - scode = 404; - } - - } else if (isAuth) { - /* Things that require authentication -- a.k.a. everything but static web app pages. */ - + if (isAuth) { if (ps[0] == "status") { - responseContentType = "application/json"; - ZT_NodeStatus status; _node->status(&status); - std::string clusterJson; + Utils::snprintf(tmp,sizeof(tmp),"%.10llx",status.address); + res["address"] = tmp; + res["publicIdentity"] = status.publicIdentity; + res["online"] = (bool)status.online; + res["tcpFallbackActive"] = _svc->tcpFallbackActive(); + res["versionMajor"] = ZEROTIER_ONE_VERSION_MAJOR; + res["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; + res["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; + res["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; + Utils::snprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); + res["version"] = tmp; + res["clock"] = OSUtils::now(); + + World planet(_node->planet()); + res["planetWorldId"] = planet.id(); + res["planetWorldTimestamp"] = planet.timestamp(); + #ifdef ZT_ENABLE_CLUSTER - { - ZT_ClusterStatus cs; - _node->clusterStatus(&cs); - - if (cs.clusterSize >= 1) { - char t[1024]; - Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members\": [",cs.myId,cs.clusterSize); - clusterJson.append(t); - for(unsigned int i=0;iclusterStatus(&cs); + if (cs.clusterSize >= 1) { + nlohmann::json cja = nlohmann::json::array(); + for(unsigned int i=0;itcpFallbackActive()) ? "true" : "false", - ZEROTIER_ONE_VERSION_MAJOR, - ZEROTIER_ONE_VERSION_MINOR, - ZEROTIER_ONE_VERSION_REVISION, - ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION, - (unsigned long long)OSUtils::now(), - ((clusterJson.length() > 0) ? clusterJson.c_str() : "null")); - responseBody = json; - scode = 200; - } else if (ps[0] == "settings") { - responseContentType = "application/json"; - responseBody = "{}"; // TODO scode = 200; + } else if (ps[0] == "moon") { + std::vector moons(_node->moons()); + if (ps.size() == 1) { + // Return [array] of all moons + + res = nlohmann::json::array(); + for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { + nlohmann::json mj; + _moonToJson(mj,*m); + res.push_back(mj); + } + + scode = 2;; + } else { + // Return a single moon by ID + + const uint64_t id = Utils::hexStrToU64(ps[1].c_str()); + for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { + if (m->id() == id) { + _moonToJson(res,*m); + scode = 200; + break; + } + } + + } } else if (ps[0] == "network") { ZT_VirtualNetworkList *nws = _node->networks(); if (nws) { if (ps.size() == 1) { // Return [array] of all networks - responseContentType = "application/json"; - responseBody = "[\n"; + + res = nlohmann::json::array(); for(unsigned long i=0;inetworkCount;++i) { - if (i > 0) - responseBody.append(","); OneService::NetworkSettings localSettings; _svc->getNetworkSettings(nws->networks[i].nwid,localSettings); - _jsonAppend(1,responseBody,&(nws->networks[i]),_svc->portDeviceName(nws->networks[i].nwid),localSettings); + nlohmann::json nj; + _networkToJson(nj,&(nws->networks[i]),_svc->portDeviceName(nws->networks[i].nwid),localSettings); + res.push_back(nj); } - responseBody.append("\n]\n"); + scode = 200; } else if (ps.size() == 2) { // Return a single network by ID or 404 if not found - uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); + + const uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); for(unsigned long i=0;inetworkCount;++i) { if (nws->networks[i].nwid == wantnw) { - responseContentType = "application/json"; OneService::NetworkSettings localSettings; _svc->getNetworkSettings(nws->networks[i].nwid,localSettings); - _jsonAppend(0,responseBody,&(nws->networks[i]),_svc->portDeviceName(nws->networks[i].nwid),localSettings); - responseBody.push_back('\n'); + _networkToJson(res,&(nws->networks[i]),_svc->portDeviceName(nws->networks[i].nwid),localSettings); scode = 200; break; } } - } // else 404 + + } else scode = 404; _node->freeQueryResult((void *)nws); } else scode = 500; } else if (ps[0] == "peer") { @@ -464,54 +328,78 @@ unsigned int ControlPlane::handleRequest( if (pl) { if (ps.size() == 1) { // Return [array] of all peers - responseContentType = "application/json"; - responseBody = "[\n"; + + res = nlohmann::json::array(); for(unsigned long i=0;ipeerCount;++i) { - if (i > 0) - responseBody.append(",\n"); - _jsonAppend(1,responseBody,&(pl->peers[i])); + nlohmann::json pj; + _peerToJson(pj,&(pl->peers[i])); + res.push_back(pj); } - responseBody.append("\n]\n"); + scode = 200; } else if (ps.size() == 2) { // Return a single peer by ID or 404 if not found + uint64_t wantp = Utils::hexStrToU64(ps[1].c_str()); for(unsigned long i=0;ipeerCount;++i) { if (pl->peers[i].address == wantp) { - responseContentType = "application/json"; - _jsonAppend(0,responseBody,&(pl->peers[i])); - responseBody.push_back('\n'); + _peerToJson(res,&(pl->peers[i])); scode = 200; break; } } - } // else 404 + + } else scode = 404; _node->freeQueryResult((void *)pl); } else scode = 500; - } else if (ps[0] == "newIdentity") { - // Return a newly generated ZeroTier identity -- this is primarily for debugging - // and testing to make it easy for automated test scripts to generate test IDs. - Identity newid; - newid.generate(); - responseBody = newid.toString(true); - responseContentType = "text/plain"; - scode = 200; } else { - if (_controller) + if (_controller) { scode = _controller->handleControlPlaneHttpGET(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); - else scode = 404; + } else scode = 404; } } else scode = 401; // isAuth == false - } else if ((httpMethod == HTTP_POST)||(httpMethod == HTTP_PUT)) { - if (isAuth) { - if (ps[0] == "settings") { - // TODO + if (ps[0] == "moon") { + if (ps.size() == 2) { + + uint64_t seed = 0; + try { + nlohmann::json j(OSUtils::jsonParse(body)); + if (j.is_object()) { + seed = OSUtils::jsonInt(j["seed"],0); + } + } catch ( ... ) { + // discard invalid JSON + } + + std::vector moons(_node->moons()); + const uint64_t id = Utils::hexStrToU64(ps[1].c_str()); + for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { + if (m->id() == id) { + _moonToJson(res,*m); + scode = 200; + break; + } + } + + if ((scode != 200)&&(seed != 0)) { + res["seed"] = seed; + res["id"] = id; + res["roots"] = nlohmann::json::array(); + res["timestamp"] = 0; + res["signature"] = nlohmann::json(); + res["updatesMustBeSignedBy"] = nlohmann::json(); + res["active"] = false; + _node->orbit(id,seed); + } + + } else scode = 404; } else if (ps[0] == "network") { if (ps.size() == 2) { + uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); _node->join(wantnw,(void *)0); // does nothing if we are a member ZT_VirtualNetworkList *nws = _node->networks(); @@ -536,17 +424,16 @@ unsigned int ControlPlane::handleRequest( } _svc->setNetworkSettings(nws->networks[i].nwid,localSettings); + _networkToJson(res,&(nws->networks[i]),_svc->portDeviceName(nws->networks[i].nwid),localSettings); - responseContentType = "application/json"; - _jsonAppend(0,responseBody,&(nws->networks[i]),_svc->portDeviceName(nws->networks[i].nwid),localSettings); - responseBody.push_back('\n'); scode = 200; break; } } _node->freeQueryResult((void *)nws); } else scode = 500; - } + + } else scode = 404; } else { if (_controller) scode = _controller->handleControlPlaneHttpPOST(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); @@ -554,12 +441,16 @@ unsigned int ControlPlane::handleRequest( } } else scode = 401; // isAuth == false - } else if (httpMethod == HTTP_DELETE) { - if (isAuth) { - if (ps[0] == "network") { + if (ps[0] == "moon") { + if (ps.size() == 2) { + _node->deorbit(Utils::hexStrToU64(ps[1].c_str())); + res["result"] = true; + scode = 200; + } // else 404 + } else if (ps[0] == "network") { ZT_VirtualNetworkList *nws = _node->networks(); if (nws) { if (ps.size() == 2) { @@ -567,8 +458,7 @@ unsigned int ControlPlane::handleRequest( for(unsigned long i=0;inetworkCount;++i) { if (nws->networks[i].nwid == wantnw) { _node->leave(wantnw,(void **)0); - responseBody = "true"; - responseContentType = "application/json"; + res["result"] = true; scode = 200; break; } @@ -582,13 +472,16 @@ unsigned int ControlPlane::handleRequest( else scode = 404; } - } else { - scode = 401; // isAuth = false - } - + } else scode = 401; // isAuth = false } else { scode = 400; - responseBody = "Method not supported."; + } + + if (responseBody.length() == 0) { + if ((res.is_object())||(res.is_array())) + responseBody = OSUtils::jsonDump(res); + else responseBody = "{}"; + responseContentType = "application/json"; } // Wrap result in jsonp function call if the user included a jsonp= url argument. diff --git a/service/ControlPlane.hpp b/service/ControlPlane.hpp index ec9b94d7..a1f743cb 100644 --- a/service/ControlPlane.hpp +++ b/service/ControlPlane.hpp @@ -40,8 +40,7 @@ struct InetAddress; class ControlPlane { public: - ControlPlane(OneService *svc,Node *n,const char *uiStaticPath); - ~ControlPlane(); + ControlPlane(OneService *svc,Node *n); /** * Set controller, which will be available under /controller @@ -88,7 +87,6 @@ private: OneService *const _svc; Node *const _node; EmbeddedNetworkController *_controller; - std::string _uiStaticPath; std::set _authTokens; Mutex _lock; }; diff --git a/service/OneService.cpp b/service/OneService.cpp index 6465463b..d56333c0 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -690,7 +690,7 @@ public: } #endif - _controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S "ui").c_str()); + _controlPlane = new ControlPlane(this,_node); _controlPlane->addAuthToken(authToken.c_str()); _controlPlane->setController(_controller); -- cgit v1.2.3 From 969e09210d89f4cecf01920d8315f984ea59245e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Feb 2017 16:14:48 -0800 Subject: Fix loading of existing moons. --- node/IncomingPacket.cpp | 2 +- node/Topology.cpp | 33 +++++++++++++++++++-------------- node/Topology.hpp | 3 ++- 3 files changed, 22 insertions(+), 16 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c6cf7f36..41a06937 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -461,7 +461,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p while (ptr < endOfWorlds) { World w; ptr += w.deserialize(*this,ptr); - RR->topology->addWorld(w); + RR->topology->addWorld(w,false); } } else { ptr += worldLen; diff --git a/node/Topology.cpp b/node/Topology.cpp index 8d0ed929..7d0b0550 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -67,7 +67,7 @@ Topology::Topology(const RuntimeEnvironment *renv) : Buffer dswtmp(buf.data(),(unsigned int)buf.length()); cachedPlanet.deserialize(dswtmp,0); } - addWorld(cachedPlanet); + addWorld(cachedPlanet,false); } catch ( ... ) {} World defaultPlanet; @@ -75,7 +75,7 @@ Topology::Topology(const RuntimeEnvironment *renv) : Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top } - addWorld(defaultPlanet); + addWorld(defaultPlanet,false); } SharedPtr Topology::addPeer(const SharedPtr &peer) @@ -273,7 +273,7 @@ bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipa return false; } -bool Topology::addWorld(const World &newWorld) +bool Topology::addWorld(const World &newWorld,bool alwaysAcceptNew) { if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON)) return false; @@ -303,15 +303,20 @@ bool Topology::addWorld(const World &newWorld) *existing = newWorld; else return false; } else if (newWorld.type() == World::TYPE_MOON) { - for(std::vector< std::pair >::iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) { - if (m->first == newWorld.id()) { - for(std::vector::const_iterator r(newWorld.roots().begin());r!=newWorld.roots().end();++r) { - if (r->identity.address() == m->second) { - _moonSeeds.erase(m); - m = _moonSeeds.end(); // cause outer loop to terminate - _moons.push_back(newWorld); - existing = &(_moons.back()); - break; + if (alwaysAcceptNew) { + _moons.push_back(newWorld); + existing = &(_moons.back()); + } else { + for(std::vector< std::pair >::iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) { + if (m->first == newWorld.id()) { + for(std::vector::const_iterator r(newWorld.roots().begin());r!=newWorld.roots().end();++r) { + if (r->identity.address() == m->second) { + _moonSeeds.erase(m); + m = _moonSeeds.end(); // cause outer loop to terminate + _moons.push_back(newWorld); + existing = &(_moons.back()); + break; + } } } } @@ -352,8 +357,8 @@ void Topology::addMoon(const uint64_t id,const Address &seed) Buffer wtmp(moonBin.data(),(unsigned int)moonBin.length()); World w; w.deserialize(wtmp); - if (w.type() == World::TYPE_MOON) { - addWorld(w); + if ((w.type() == World::TYPE_MOON)&&(w.id() == id)) { + addWorld(w,true); return; } } diff --git a/node/Topology.hpp b/node/Topology.hpp index 35f98ccc..39367d6e 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -258,9 +258,10 @@ public: * Validate new world and update if newer and signature is okay * * @param newWorld A new or updated planet or moon to learn + * @param alwaysAcceptNew If true, always accept new moons even if we're not waiting for one * @return True if it was valid and newer than current (or totally new for moons) */ - bool addWorld(const World &newWorld); + bool addWorld(const World &newWorld,bool alwaysAcceptNew); /** * Add a moon -- cgit v1.2.3 From af4e79735c3f97d4228472077bcd5d2ddfb2cb93 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Feb 2017 16:38:21 -0800 Subject: Fix "orbit" semantics. Federation works. --- node/IncomingPacket.cpp | 8 ++++---- node/Topology.cpp | 15 ++++++++++++++- node/Topology.hpp | 6 ++++++ 3 files changed, 24 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 41a06937..b077f7e2 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -455,16 +455,16 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p // Handle planet or moon updates if present if ((ptr + 2) <= size()) { - const unsigned int worldLen = at(ptr); ptr += 2; - if (RR->topology->isUpstream(peer->identity())) { - const unsigned int endOfWorlds = ptr + worldLen; + const unsigned int worldsLen = at(ptr); ptr += 2; + if (RR->topology->shouldAcceptWorldUpdateFrom(peer->address())) { + const unsigned int endOfWorlds = ptr + worldsLen; while (ptr < endOfWorlds) { World w; ptr += w.deserialize(*this,ptr); RR->topology->addWorld(w,false); } } else { - ptr += worldLen; + ptr += worldsLen; } } diff --git a/node/Topology.cpp b/node/Topology.cpp index 7d0b0550..5abc4df0 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -225,6 +225,18 @@ bool Topology::isUpstream(const Identity &id) const return (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),id.address()) != _upstreamAddresses.end()); } +bool Topology::shouldAcceptWorldUpdateFrom(const Address &addr) const +{ + Mutex::Lock _l(_upstreams_m); + if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),addr) != _upstreamAddresses.end()) + return true; + for(std::vector< std::pair< uint64_t,Address> >::const_iterator s(_moonSeeds.begin());s!=_moonSeeds.end();++s) { + if (s->second == addr) + return true; + } + return false; +} + ZT_PeerRole Topology::role(const Address &ztaddr) const { Mutex::Lock _l(_upstreams_m); @@ -312,12 +324,13 @@ bool Topology::addWorld(const World &newWorld,bool alwaysAcceptNew) for(std::vector::const_iterator r(newWorld.roots().begin());r!=newWorld.roots().end();++r) { if (r->identity.address() == m->second) { _moonSeeds.erase(m); - m = _moonSeeds.end(); // cause outer loop to terminate _moons.push_back(newWorld); existing = &(_moons.back()); break; } } + if (existing) + break; } } } diff --git a/node/Topology.hpp b/node/Topology.hpp index 39367d6e..37615b49 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -147,6 +147,12 @@ public: */ bool isUpstream(const Identity &id) const; + /** + * @param addr Address to check + * @return True if we should accept a world update from this address + */ + bool shouldAcceptWorldUpdateFrom(const Address &addr) const; + /** * @param ztaddr ZeroTier address * @return Peer role for this device -- cgit v1.2.3 From afba19e01cc5db2030059485217e5fddadfa3ee1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 16 Feb 2017 09:44:04 -0800 Subject: When deciding whether to send PUSH_DIRECT_PATHS we should check global trust flag, not the one passed into receive(). --- node/Peer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index d5847092..25efab42 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -207,7 +207,7 @@ void Peer::received( path->sent(now); } } - } else if (trustEstablished) { + } else if (this->trustEstablished(now)) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) #ifdef ZT_ENABLE_CLUSTER // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection -- cgit v1.2.3 From b679ebde3b05efeb3346e20dfb216cf3b3bc2b1d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 22 Feb 2017 15:32:55 -0800 Subject: Ad-hoc networks, a cool and easy to implement little feature that allows controllerless networks. These only allow IPv6 6plane, no multicast, and the network ID encodes the allowed port range. --- include/ZeroTierOne.h | 12 ++++----- node/Network.cpp | 66 ++++++++++++++++++++++++++++++++++++++++++++++ rule-compiler/package.json | 2 +- 3 files changed, 73 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 9690489a..e2380a7b 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -610,15 +610,15 @@ enum ZT_VirtualNetworkRuleType typedef struct { /** - * Least significant 7 bits: ZT_VirtualNetworkRuleType, most significant 1 bit is NOT bit + * Type and flags * - * If the NOT bit is set, then matches will be interpreted as "does not - * match." The NOT bit has no effect on actions. + * Bits are: NOTTTTTT * - * Use "& 0x7f" to get the enum and "& 0x80" to get the NOT flag. + * N - If true, sense of match is inverted (no effect on actions) + * O - If true, result is ORed with previous instead of ANDed (no effect on actions) + * T - Rule or action type * - * The union 'v' is a variant type, and this selects which field in 'v' is - * actually used and valid. + * AND with 0x3f to get type, 0x80 to get NOT bit, and 0x40 to get OR bit. */ uint8_t t; diff --git a/node/Network.cpp b/node/Network.cpp index 461e1c20..290ceaf9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1099,6 +1099,72 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) void Network::requestConfiguration() { + /* ZeroTier addresses can't begin with 0xff, so this is used to mark controllerless + * network IDs. Controllerless network IDs only support unicast IPv6 using the 6plane + * addressing scheme and have the following format: 0xffSSSSEEEE000000 where SSSS + * is the 16-bit starting IP port range allowed and EEEE is the 16-bit ending IP port + * range allowed. Remaining digits are reserved for future use and must be zero. */ + if ((_id >> 56) == 0xff) { + const uint16_t startPortRange = (uint16_t)((_id >> 40) & 0xffff); + const uint16_t endPortRange = (uint16_t)((_id >> 24) & 0xffff); + if (((_id & 0xffffff) == 0)&&(endPortRange >= startPortRange)) { + NetworkConfig *const nconf = new NetworkConfig(); + + nconf->networkId = _id; + nconf->timestamp = RR->node->now(); + nconf->credentialTimeMaxDelta = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + nconf->revision = 1; + nconf->issuedTo = RR->identity.address(); + nconf->flags = ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + nconf->staticIpCount = 1; + nconf->ruleCount = 14; + nconf->staticIps[0] = InetAddress::makeIpv66plane(_id,RR->identity.address().toInt()); + + // Drop everything but IPv6 + nconf->rules[0].t = (uint8_t)ZT_NETWORK_RULE_MATCH_ETHERTYPE | 0x80; // NOT + nconf->rules[0].v.etherType = 0x86dd; // IPv6 + nconf->rules[1].t = (uint8_t)ZT_NETWORK_RULE_ACTION_DROP; + + // Allow ICMPv6 + nconf->rules[2].t = (uint8_t)ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; + nconf->rules[2].v.ipProtocol = 0x3a; // ICMPv6 + nconf->rules[3].t = (uint8_t)ZT_NETWORK_RULE_ACTION_ACCEPT; + + // Allow destination ports within range + nconf->rules[4].t = (uint8_t)ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; + nconf->rules[4].v.ipProtocol = 0x11; // UDP + nconf->rules[5].t = (uint8_t)ZT_NETWORK_RULE_MATCH_IP_PROTOCOL | 0x40; // OR + nconf->rules[5].v.ipProtocol = 0x06; // TCP + nconf->rules[6].t = (uint8_t)ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; + nconf->rules[6].v.port[0] = startPortRange; + nconf->rules[6].v.port[1] = endPortRange; + nconf->rules[7].t = (uint8_t)ZT_NETWORK_RULE_ACTION_ACCEPT; + + // Allow non-SYN TCP packets to permit non-connection-initiating traffic + nconf->rules[8].t = (uint8_t)ZT_NETWORK_RULE_MATCH_CHARACTERISTICS | 0x80; // NOT + nconf->rules[8].v.characteristics = ZT_RULE_PACKET_CHARACTERISTICS_TCP_SYN; + nconf->rules[9].t = (uint8_t)ZT_NETWORK_RULE_ACTION_ACCEPT; + + // Also allow SYN+ACK which are replies to SYN + nconf->rules[10].t = (uint8_t)ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; + nconf->rules[10].v.characteristics = ZT_RULE_PACKET_CHARACTERISTICS_TCP_SYN; + nconf->rules[11].t = (uint8_t)ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; + nconf->rules[11].v.characteristics = ZT_RULE_PACKET_CHARACTERISTICS_TCP_ACK; + nconf->rules[12].t = (uint8_t)ZT_NETWORK_RULE_ACTION_ACCEPT; + + nconf->rules[13].t = (uint8_t)ZT_NETWORK_RULE_ACTION_DROP; + + nconf->type = ZT_NETWORK_TYPE_PUBLIC; + Utils::snprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); + + this->setConfiguration(*nconf,false); + delete nconf; + } else { + this->setNotFound(); + } + return; + } + const Address ctrl(controller()); Dictionary rmd; diff --git a/rule-compiler/package.json b/rule-compiler/package.json index 6fe70696..16f7a23f 100644 --- a/rule-compiler/package.json +++ b/rule-compiler/package.json @@ -1,6 +1,6 @@ { "name": "zerotier-rule-compiler", - "version": "1.1.17-0", + "version": "1.1.17-1", "description": "ZeroTier Rule Script Compiler", "main": "cli.js", "scripts": { -- cgit v1.2.3 From 10185e92faa77a4b032a27a7c01b4186727b91b9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 23 Feb 2017 11:47:36 -0800 Subject: Certificate of ownership -- used to secure against IP address spoofing, especially for IPv4 and regular IPv6. --- controller/EmbeddedNetworkController.cpp | 9 ++ include/ZeroTierOne.h | 5 + node/Capability.hpp | 1 - node/CertificateOfOwnership.cpp | 46 ++++++ node/CertificateOfOwnership.hpp | 251 +++++++++++++++++++++++++++++++ node/IncomingPacket.cpp | 19 +++ node/Membership.cpp | 164 +++++++++++++++----- node/Membership.hpp | 81 +++++----- node/Network.hpp | 11 ++ node/NetworkConfig.cpp | 25 ++- node/NetworkConfig.hpp | 21 ++- node/Packet.hpp | 2 + node/Revocation.hpp | 5 +- node/Tag.hpp | 3 +- objects.mk | 1 + 15 files changed, 550 insertions(+), 94 deletions(-) create mode 100644 node/CertificateOfOwnership.cpp create mode 100644 node/CertificateOfOwnership.hpp (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index ca548fd4..78a9b7c7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1706,6 +1706,15 @@ void EmbeddedNetworkController::_request( } } + // Issue a certificate of ownership for all static IPs + if (nc.staticIpCount) { + nc.certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); + for(unsigned int i=0;i(p); p += 8; _ts = b.template at(p); p += 8; _id = b.template at(p); p += 4; diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp new file mode 100644 index 00000000..8305c489 --- /dev/null +++ b/node/CertificateOfOwnership.cpp @@ -0,0 +1,46 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "CertificateOfOwnership.hpp" +#include "RuntimeEnvironment.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Switch.hpp" +#include "Network.hpp" + +namespace ZeroTier { + +int CertificateOfOwnership::verify(const RuntimeEnvironment *RR) const +{ + if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) + return -1; + const Identity id(RR->topology->getIdentity(_signedBy)); + if (!id) { + RR->sw->requestWhois(_signedBy); + return 1; + } + try { + Buffer<(sizeof(CertificateOfOwnership) + 64)> tmp; + this->serialize(tmp,true); + return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1); + } catch ( ... ) { + return -1; + } +} + +} // namespace ZeroTier diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp new file mode 100644 index 00000000..69b26aec --- /dev/null +++ b/node/CertificateOfOwnership.hpp @@ -0,0 +1,251 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CERTIFICATEOFOWNERSHIP_HPP +#define ZT_CERTIFICATEOFOWNERSHIP_HPP + +#include +#include +#include +#include + +#include "Constants.hpp" +#include "C25519.hpp" +#include "Address.hpp" +#include "Identity.hpp" +#include "Buffer.hpp" +#include "InetAddress.hpp" +#include "MAC.hpp" + +// Max things per CertificateOfOwnership +#define ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS 16 + +// Maximum size of a thing's value field in bytes +#define ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE 16 + +namespace ZeroTier { + +class RuntimeEnvironment; + +/** + * Certificate indicating ownership of a network identifier + */ +class CertificateOfOwnership +{ +public: + enum Thing + { + THING_NULL = 0, + THING_MAC_ADDRESS = 1, + THING_IPV4_ADDRESS = 2, + THING_IPV6_ADDRESS = 3 + }; + + CertificateOfOwnership() : + _networkId(0), + _ts(0), + _id(0), + _thingCount(0) + { + } + + CertificateOfOwnership(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id) : + _networkId(nwid), + _ts(ts), + _flags(0), + _id(id), + _thingCount(0), + _issuedTo(issuedTo) + { + } + + inline uint64_t networkId() const { return _networkId; } + inline uint64_t timestamp() const { return _ts; } + inline uint32_t id() const { return _id; } + inline unsigned int thingCount() const { return (unsigned int)_thingCount; } + + inline Thing thingType(const unsigned int i) const { return (Thing)_thingTypes[i]; } + inline const uint8_t *thingValue(const unsigned int i) const { return _thingValues[i]; } + + inline const Address &issuedTo() const { return _issuedTo; } + + inline bool owns(const Thing &t,const void *v,unsigned int l) + { + for(unsigned int i=0,j=_thingCount;i(v)[k] != _thingValues[i][k]) + break; + ++k; + } + if (k == l) + return true; + } + } + return false; + } + + inline bool owns(const InetAddress &ip) + { + if (ip.ss_family == AF_INET) + return this->owns(THING_IPV4_ADDRESS,&(reinterpret_cast(&ip)->sin_addr.s_addr),4); + if (ip.ss_family == AF_INET6) + return this->owns(THING_IPV6_ADDRESS,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + return false; + } + + inline bool owns(const MAC &mac) + { + uint8_t tmp[6]; + mac.copyTo(tmp,6); + return this->owns(THING_MAC_ADDRESS,tmp,6); + } + + inline void addThing(const InetAddress &ip) + { + if (_thingCount >= ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS) return; + if (ip.ss_family == AF_INET) { + _thingTypes[_thingCount] = THING_IPV4_ADDRESS; + memcpy(_thingValues[_thingCount],&(reinterpret_cast(&ip)->sin_addr.s_addr),4); + ++_thingCount; + } else if (ip.ss_family == AF_INET6) { + _thingTypes[_thingCount] = THING_IPV6_ADDRESS; + memcpy(_thingValues[_thingCount],reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + ++_thingCount; + } + } + + inline void addThing(const MAC &mac) + { + if (_thingCount >= ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS) return; + _thingTypes[_thingCount] = THING_MAC_ADDRESS; + mac.copyTo(_thingValues[_thingCount],6); + ++_thingCount; + } + + /** + * @param signer Signing identity, must have private key + * @return True if signature was successful + */ + inline bool sign(const Identity &signer) + { + if (signer.hasPrivate()) { + Buffer tmp; + _signedBy = signer.address(); + this->serialize(tmp,true); + _signature = signer.sign(tmp.data(),tmp.size()); + return true; + } + return false; + } + + /** + * @param RR Runtime environment to allow identity lookup for signedBy + * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature + */ + int verify(const RuntimeEnvironment *RR) const; + + template + inline void serialize(Buffer &b,const bool forSign = false) const + { + if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + + b.append(_networkId); + b.append(_ts); + b.append(_flags); + b.append(_id); + b.append((uint16_t)_thingCount); + for(unsigned int i=0,j=_thingCount;i + inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) + { + unsigned int p = startAt; + + memset(this,0,sizeof(CertificateOfOwnership)); + + _networkId = b.template at(p); p += 8; + _ts = b.template at(p); p += 8; + _flags = b.template at(p); p += 8; + _id = b.template at(p); p += 4; + _thingCount = b.template at(p); p += 2; + for(unsigned int i=0,j=_thingCount;i(p) != ZT_C25519_SIGNATURE_LEN) + throw std::runtime_error("invalid signature length"); + p += 2; + memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; + } else { + p += 2 + b.template at(p); + } + + p += 2 + b.template at(p); + if (p > b.size()) + throw std::runtime_error("extended field overflow"); + + return (p - startAt); + } + + // Provides natural sort order by ID + inline bool operator<(const CertificateOfOwnership &coo) const { return (_id < coo._id); } + + inline bool operator==(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) == 0); } + inline bool operator!=(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) != 0); } + +private: + uint64_t _networkId; + uint64_t _ts; + uint64_t _flags; + uint32_t _id; + uint16_t _thingCount; + uint8_t _thingTypes[ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS]; + uint8_t _thingValues[ZT_CERTIFICATEOFOWNERSHIP_MAX_THINGS][ZT_CERTIFICATEOFOWNERSHIP_MAX_THING_VALUE_SIZE]; + Address _issuedTo; + Address _signedBy; + C25519::Signature _signature; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b077f7e2..b5b2bcb3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -832,6 +832,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S Capability cap; Tag tag; Revocation revocation; + CertificateOfOwnership coo; bool trustEstablished = false; unsigned int p = ZT_PACKET_IDX_PAYLOAD; @@ -909,6 +910,24 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } } + + const unsigned int numCoos = at(p); p += 2; + for(unsigned int i=0;i network(RR->node->network(coo.networkId())); + if (network) { + switch(network->addCredential(coo)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; + } + } + } } peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); diff --git a/node/Membership.cpp b/node/Membership.cpp index 8c6dab64..1eacb93d 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -37,6 +37,7 @@ Membership::Membership() : { for(unsigned int i=0;i= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCoos[c].lastPushed = now; + _localCoos[c].id = nconf.certificatesOfOwnership[c].id(); + sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); + } + } + unsigned int tagPtr = 0; - while ((tagPtr < sendTagCount)||(sendCom)||(sendCap)) { + unsigned int cooPtr = 0; + while ((tagPtr < sendTagCount)||(cooPtr < sendCooCount)||(sendCom)||(sendCap)) { Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); if (sendCom) { @@ -82,7 +94,7 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now const unsigned int tagCountAt = outp.size(); outp.addSize(2); unsigned int thisPacketTagCount = 0; - while ((tagPtr < sendTagCount)&&((outp.size() + sizeof(Tag) + 32) < ZT_PROTO_MAX_PACKET_LENGTH)) { + while ((tagPtr < sendTagCount)&&((outp.size() + sizeof(Tag) + 16) < ZT_PROTO_MAX_PACKET_LENGTH)) { sendTags[tagPtr++]->serialize(outp); ++thisPacketTagCount; } @@ -91,6 +103,15 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now // No revocations, these propagate differently outp.append((uint16_t)0); + const unsigned int cooCountAt = outp.size(); + outp.addSize(2); + unsigned int thisPacketCooCount = 0; + while ((cooPtr < sendCooCount)&&((outp.size() + sizeof(CertificateOfOwnership) + 16) < ZT_PROTO_MAX_PACKET_LENGTH)) { + sendCoos[cooPtr++]->serialize(outp); + ++thisPacketCooCount; + } + outp.setAt(cooCountAt,(uint16_t)thisPacketCooCount); + outp.compress(); RR->sw->send(outp,true); } @@ -98,14 +119,14 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now const Capability *Membership::getCapability(const NetworkConfig &nconf,const uint32_t id) const { - const _RemoteCapability *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialSorter<_RemoteCapability>()); - return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->cap,**c))) ? &((*c)->cap) : (const Capability *)0) : (const Capability *)0); + const _RemoteCredential *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialComp()); + return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->credential,**c))) ? &((*c)->credential) : (const Capability *)0) : (const Capability *)0); } const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) const { - const _RemoteTag *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialSorter<_RemoteTag>()); - return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->tag,**t))) ? &((*t)->tag) : (const Tag *)0) : (const Tag *)0); + const _RemoteCredential *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialComp()); + return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->credential,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) @@ -141,14 +162,14 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag) { - _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialSorter<_RemoteTag>()); - _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteTag *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->tag.timestamp() > tag.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->credential.timestamp() > tag.timestamp()) ) { TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_REJECTED; } - if (have->tag == tag) { + if (have->credential == tag) { TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_ACCEPTED_REDUNDANT; } @@ -162,7 +183,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); if (!have) have = _newTag(tag.id()); have->lastReceived = RR->node->now(); - have->tag = tag; + have->credential = tag; return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -171,14 +192,14 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap) { - _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialSorter<_RemoteCapability>()); - _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCapability *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->cap.timestamp() > cap.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->credential.timestamp() > cap.timestamp()) ) { TRACE("addCredential(Capability) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; } - if (have->cap == cap) { + if (have->credential == cap) { TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_ACCEPTED_REDUNDANT; } @@ -192,7 +213,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); if (!have) have = _newCapability(cap.id()); have->lastReceived = RR->node->now(); - have->cap = cap; + have->credential = cap; return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -209,13 +230,15 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme switch(rev.type()) { default: //case Revocation::CREDENTIAL_TYPE_ALL: - return ( (_revokeCom(rev)||_revokeCap(rev,now)||_revokeTag(rev,now)) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT ); + return ( (_revokeCom(rev)||_revokeCap(rev,now)||_revokeTag(rev,now)||_revokeCoo(rev,now)) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT ); case Revocation::CREDENTIAL_TYPE_COM: return (_revokeCom(rev) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); case Revocation::CREDENTIAL_TYPE_CAPABILITY: return (_revokeCap(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); case Revocation::CREDENTIAL_TYPE_TAG: return (_revokeTag(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); + case Revocation::CREDENTIAL_TYPE_COO: + return (_revokeCoo(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); } } case 1: @@ -223,9 +246,40 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::_RemoteTag *Membership::_newTag(const uint64_t id) + +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo) +{ + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; + if (have) { + if ( (!_isCredentialTimestampValid(nconf,coo,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); + return ADD_REJECTED; + } + if (have->credential == coo) { + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); + return ADD_ACCEPTED_REDUNDANT; + } + } + + switch(coo.verify(RR)) { + default: + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); + return ADD_REJECTED; + case 0: + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); + if (!have) have = _newCoo(coo.id()); + have->lastReceived = RR->node->now(); + have->credential = coo; + return ADD_ACCEPTED_NEW; + case 1: + return ADD_DEFERRED_FOR_WHOIS; + } +} + +Membership::_RemoteCredential *Membership::_newTag(const uint64_t id) { - _RemoteTag *t = NULL; + _RemoteCredential *t = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -236,21 +290,21 @@ Membership::_RemoteTag *Membership::_newTag(const uint64_t id) minlr = _remoteTags[i]->lastReceived; } } - - if (t) { - t->id = id; - t->lastReceived = 0; - t->revocationThreshold = 0; - t->tag = Tag(); - } - - std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialSorter<_RemoteTag>()); + + if (t) { + t->id = id; + t->lastReceived = 0; + t->revocationThreshold = 0; + t->credential = Tag(); + } + + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS])); return t; } -Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) +Membership::_RemoteCredential *Membership::_newCapability(const uint64_t id) { - _RemoteCapability *c = NULL; + _RemoteCredential *c = NULL; uint64_t minlr = 0xffffffffffffffffULL; for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { @@ -266,10 +320,35 @@ Membership::_RemoteCapability *Membership::_newCapability(const uint64_t id) c->id = id; c->lastReceived = 0; c->revocationThreshold = 0; - c->cap = Capability(); + c->credential = Capability(); + } + + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES])); + return c; +} + +Membership::_RemoteCredential *Membership::_newCoo(const uint64_t id) +{ + _RemoteCredential *c = NULL; + uint64_t minlr = 0xffffffffffffffffULL; + for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { + c = _remoteCoos[i]; + break; + } else if (_remoteCoos[i]->lastReceived <= minlr) { + c = _remoteCoos[i]; + minlr = _remoteCoos[i]->lastReceived; + } + } + + if (c) { + c->id = id; + c->lastReceived = 0; + c->revocationThreshold = 0; + c->credential = CertificateOfOwnership(); } - std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialSorter<_RemoteCapability>()); + std::sort(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP])); return c; } @@ -284,8 +363,8 @@ bool Membership::_revokeCom(const Revocation &rev) bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) { - _RemoteCapability *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialSorter<_RemoteCapability>()); - _RemoteCapability *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCapability *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; if (!have) have = _newCapability(rev.credentialId()); if (rev.threshold() > have->revocationThreshold) { have->lastReceived = now; @@ -297,8 +376,8 @@ bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) { - _RemoteTag *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialSorter<_RemoteTag>()); - _RemoteTag *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteTag *)0; + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; if (!have) have = _newTag(rev.credentialId()); if (rev.threshold() > have->revocationThreshold) { have->lastReceived = now; @@ -308,4 +387,17 @@ bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) return false; } +bool Membership::_revokeCoo(const Revocation &rev,const uint64_t now) +{ + _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); + _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; + if (!have) have = _newCoo(rev.credentialId()); + if (rev.threshold() > have->revocationThreshold) { + have->lastReceived = now; + have->revocationThreshold = rev.threshold(); + return true; + } + return false; +} + } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 9814dce8..4e9d7769 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -39,49 +39,30 @@ class Network; /** * A container for certificates of membership and other network credentials * - * This is kind of analogous to a join table between Peer and Network. It is - * held by the Network object for each participating Peer. + * This is essentially a relational join between Peer and Network. * * This class is not thread safe. It must be locked externally. */ class Membership { private: - // Tags and related state - struct _RemoteTag - { - _RemoteTag() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} - // Tag ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) - uint64_t id; - // Last time we received THEIR tag (with this ID) - uint64_t lastReceived; - // Revocation blacklist threshold or 0 if none - uint64_t revocationThreshold; - // THEIR tag - Tag tag; - }; - - // Credentials and related state - struct _RemoteCapability + template + struct _RemoteCredential { - _RemoteCapability() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} - // Capability ID (last 32 bits, first 32 bits are set in unused entries to sort them to end) + _RemoteCredential() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} uint64_t id; - // Last time we received THEIR capability (with this ID) - uint64_t lastReceived; - // Revocation blacklist threshold or 0 if none - uint64_t revocationThreshold; - // THEIR capability - Capability cap; + uint64_t lastReceived; // last time we got this credential + uint64_t revocationThreshold; // credentials before this time are invalid + T credential; + inline bool operator<(const _RemoteCredential &c) const { return (id < c.id); } }; - // Comparison operator for remote credential entries template - struct _RemoteCredentialSorter + struct _RemoteCredentialComp { - inline bool operator()(const T *a,const T *b) const { return (a->id < b->id); } - inline bool operator()(const uint64_t a,const T *b) const { return (a < b->id); } - inline bool operator()(const T *a,const uint64_t b) const { return (a->id < b); } + inline bool operator()(const _RemoteCredential *a,const _RemoteCredential *b) const { return (a->id < b->id); } + inline bool operator()(const uint64_t a,const _RemoteCredential *b) const { return (a < b->id); } + inline bool operator()(const _RemoteCredential *a,const uint64_t b) const { return (a->id < b); } inline bool operator()(const uint64_t a,const uint64_t b) const { return (a < b); } }; @@ -89,8 +70,8 @@ private: struct _LocalCredentialPushState { _LocalCredentialPushState() : lastPushed(0),id(0) {} - uint64_t lastPushed; - uint32_t id; + uint64_t lastPushed; // last time we sent our own copy of this credential + uint64_t id; }; public: @@ -117,7 +98,7 @@ public: { for(;;) { if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Capability *tmp = &((*_i)->cap); + const Capability *tmp = &((*_i)->credential); if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { ++_i; return tmp; @@ -131,7 +112,7 @@ public: private: const Membership *_m; const NetworkConfig *_c; - const _RemoteCapability *const *_i; + const _RemoteCredential *const *_i; }; friend class CapabilityIterator; @@ -150,7 +131,7 @@ public: { for(;;) { if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Tag *tmp = &((*_i)->tag); + const Tag *tmp = &((*_i)->credential); if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { ++_i; return tmp; @@ -164,7 +145,7 @@ public: private: const Membership *_m; const NetworkConfig *_c; - const _RemoteTag *const *_i; + const _RemoteCredential *const *_i; }; friend class TagIterator; @@ -249,12 +230,19 @@ public: */ AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev); + /** + * Validate and add a credential if signature is okay and it's otherwise good + */ + AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo); + private: - _RemoteTag *_newTag(const uint64_t id); - _RemoteCapability *_newCapability(const uint64_t id); + _RemoteCredential *_newTag(const uint64_t id); + _RemoteCredential *_newCapability(const uint64_t id); + _RemoteCredential *_newCoo(const uint64_t id); bool _revokeCom(const Revocation &rev); bool _revokeCap(const Revocation &rev,const uint64_t now); bool _revokeTag(const Revocation &rev,const uint64_t now); + bool _revokeCoo(const Revocation &rev,const uint64_t now); template inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred,const CS &state) const @@ -275,17 +263,20 @@ private: // Remote member's latest network COM CertificateOfMembership _com; - // Sorted (in ascending order of ID) arrays of pointers to remote tags and capabilities - _RemoteTag *_remoteTags[ZT_MAX_NETWORK_TAGS]; - _RemoteCapability *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; + // Sorted (in ascending order of ID) arrays of pointers to remote credentials + _RemoteCredential *_remoteTags[ZT_MAX_NETWORK_TAGS]; + _RemoteCredential *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; + _RemoteCredential *_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; - // This is the RAM allocated for remote tags and capabilities from which the sorted arrays are populated - _RemoteTag _tagMem[ZT_MAX_NETWORK_TAGS]; - _RemoteCapability _capMem[ZT_MAX_NETWORK_CAPABILITIES]; + // This is the RAM allocated for remote credential cache objects + _RemoteCredential _tagMem[ZT_MAX_NETWORK_TAGS]; + _RemoteCredential _capMem[ZT_MAX_NETWORK_CAPABILITIES]; + _RemoteCredential _cooMem[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; // Local credential push state tracking _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; + _LocalCredentialPushState _localCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; }; } // namespace ZeroTier diff --git a/node/Network.hpp b/node/Network.hpp index 85ee6e9a..56c7fc60 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -301,6 +301,17 @@ public: */ Membership::AddCredentialResult addCredential(const Address &sentFrom,const Revocation &rev); + /** + * Validate a credential and learn it if it passes certificate and other checks + */ + inline Membership::AddCredentialResult addCredential(const CertificateOfOwnership &coo) + { + if (coo.networkId() != _id) + return Membership::ADD_REJECTED; + Mutex::Lock _l(_lock); + return _membership(coo.issuedTo()).addCredential(RR,_config,coo); + } + /** * Force push credentials (COM, etc.) to a peer now * diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 2f356b15..fe7393e8 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -21,7 +21,6 @@ #include #include "NetworkConfig.hpp" -#include "Utils.hpp" namespace ZeroTier { @@ -137,6 +136,13 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TAGS,*tmp)) return false; } + tmp->clear(); + for(unsigned int i=0;icertificateOfOwnershipCount;++i) + this->certificatesOfOwnership[i].serialize(*tmp); + if (tmp->size()) { + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP,*tmp)) return false; + } + tmp->clear(); for(unsigned int i=0;ispecialistCount;++i) tmp->append((uint64_t)this->specialists[i]); @@ -297,10 +303,23 @@ bool NetworkConfig::fromDictionary(const Dictionarytags[0]),&(this->tags[this->tagCount])); } + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP,*tmp)) { + unsigned int p = 0; + while (p < tmp->size()) { + if (certificateOfOwnershipCount < ZT_MAX_CERTIFICATES_OF_OWNERSHIP) + p += certificatesOfOwnership[certificateOfOwnershipCount++].deserialize(*tmp,p); + else { + CertificateOfOwnership foo; + p += foo.deserialize(*tmp,p); + } + } + } + if (d.get(ZT_NETWORKCONFIG_DICT_KEY_SPECIALISTS,*tmp)) { unsigned int p = 0; - while (((p + 8) <= tmp->size())&&(specialistCount < ZT_MAX_NETWORK_SPECIALISTS)) { - this->specialists[this->specialistCount++] = tmp->at(p); + while ((p + 8) <= tmp->size()) { + if (specialistCount < ZT_MAX_NETWORK_SPECIALISTS) + this->specialists[this->specialistCount++] = tmp->at(p); p += 8; } } diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 39087395..85c24090 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -35,10 +35,12 @@ #include "MulticastGroup.hpp" #include "Address.hpp" #include "CertificateOfMembership.hpp" +#include "CertificateOfOwnership.hpp" #include "Capability.hpp" #include "Tag.hpp" #include "Dictionary.hpp" #include "Identity.hpp" +#include "Utils.hpp" /** * Default maximum time delta for COMs, tags, and capabilities @@ -99,7 +101,7 @@ namespace ZeroTier { // Dictionary capacity needed for max size network config -#define ZT_NETWORKCONFIG_DICT_CAPACITY (4096 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS)) +#define ZT_NETWORKCONFIG_DICT_CAPACITY (1024 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP)) // Dictionary capacity needed for max size network meta-data #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 1024 @@ -173,6 +175,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_CAPABILITIES "CAP" // tags (binary blobs) #define ZT_NETWORKCONFIG_DICT_KEY_TAGS "TAG" +// tags (binary blobs) +#define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP "COO" // curve25519 signature #define ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE "C25519" @@ -473,11 +477,6 @@ public: */ unsigned int staticIpCount; - /** - * Number of pinned devices (devices with physical address hints) - */ - unsigned int pinnedCount; - /** * Number of rule table entries */ @@ -493,6 +492,11 @@ public: */ unsigned int tagCount; + /** + * Number of certificates of ownership + */ + unsigned int certificateOfOwnershipCount; + /** * Specialist devices * @@ -526,6 +530,11 @@ public: */ Tag tags[ZT_MAX_NETWORK_TAGS]; + /** + * Certificates of ownership for this network member + */ + CertificateOfOwnership certificatesOfOwnership[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + /** * Network type (currently just public or private) */ diff --git a/node/Packet.hpp b/node/Packet.hpp index b736b84a..6482356a 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -730,6 +730,8 @@ public: * <[...] one or more serialized Tags> * <[2] 16-bit number of revocations> * <[...] one or more serialized Revocations> + * <[2] 16-bit number of certificates of ownership> + * <[...] one or more serialized CertificateOfOwnership> * * This can be sent by anyone at any time to push network credentials. * These will of course only be accepted if they are properly signed. diff --git a/node/Revocation.hpp b/node/Revocation.hpp index bc290e75..3903f440 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -50,9 +50,10 @@ public: enum CredentialType { CREDENTIAL_TYPE_ALL = 0, - CREDENTIAL_TYPE_COM = 1, + CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership CREDENTIAL_TYPE_CAPABILITY = 2, - CREDENTIAL_TYPE_TAG = 3 + CREDENTIAL_TYPE_TAG = 3, + CREDENTIAL_TYPE_COO = 4 // CertificateOfOwnership }; Revocation() diff --git a/node/Tag.hpp b/node/Tag.hpp index 65348200..146e8da9 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -139,7 +139,8 @@ public: { unsigned int p = startAt; - // These are the same between Tag and Capability + memset(this,0,sizeof(Tag)); + _networkId = b.template at(p); p += 8; _ts = b.template at(p); p += 8; _id = b.template at(p); p += 4; diff --git a/objects.mk b/objects.mk index 31498b72..427024eb 100644 --- a/objects.mk +++ b/objects.mk @@ -4,6 +4,7 @@ OBJS=\ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ + node/CertificateOfOwnership.o \ node/Cluster.o \ node/Identity.o \ node/IncomingPacket.o \ -- cgit v1.2.3 From 93ec86a26e5e9312ee913258c3e55eb03af147a7 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 23 Feb 2017 12:26:11 -0800 Subject: iOS fixes --- include/ZeroTierOne.h | 2 +- node/Membership.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index c1dbd8f8..90867162 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1798,7 +1798,7 @@ enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moo * @param moonWorldId World ID of moon to remove * @return Error if anything bad happened */ -ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId); +enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId); /** * Get this node's 40-bit ZeroTier address diff --git a/node/Membership.cpp b/node/Membership.cpp index 1eacb93d..c021cc76 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -253,21 +253,21 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; if (have) { if ( (!_isCredentialTimestampValid(nconf,coo,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",coo.issuedTo().toString().c_str(),coo.networkId()); return ADD_REJECTED; } if (have->credential == coo) { - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (redundant)",coo.issuedTo().toString().c_str(),coo.networkId()); return ADD_ACCEPTED_REDUNDANT; } } switch(coo.verify(RR)) { default: - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",coo.issuedTo().toString().c_str(),coo.networkId()); return ADD_REJECTED; case 0: - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); + TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (new)",coo.issuedTo().toString().c_str(),coo.networkId()); if (!have) have = _newCoo(coo.id()); have->lastReceived = RR->node->now(); have->credential = coo; -- cgit v1.2.3 From 72653e54f951b2a47686d420186f59f533542940 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 23 Feb 2017 12:34:17 -0800 Subject: Finish wiring up ipauth and macauth to Network filter. --- include/ZeroTierOne.h | 10 ++++++++++ node/CertificateOfOwnership.cpp | 17 +++++++++++++++++ node/CertificateOfOwnership.hpp | 29 +++++++---------------------- node/Membership.cpp | 20 +++++++------------- node/Membership.hpp | 35 +++++++++++++++++++++++++---------- node/Network.cpp | 25 +++++++++++++++++++++++++ rule-compiler/package.json | 2 +- rule-compiler/rule-compiler.js | 2 ++ 8 files changed, 94 insertions(+), 46 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index c1dbd8f8..ddd0b404 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -194,6 +194,16 @@ extern "C" { */ #define ZT_RULE_PACKET_CHARACTERISTICS_BROADCAST 0x2000000000000000ULL +/** + * Packet characteristics flag: sending IP address has a certificate of ownership + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED 0x1000000000000000ULL + +/** + * Packet characteristics flag: sending MAC address has a certificate of ownership + */ +#define ZT_RULE_PACKET_CHARACTERISTICS_SENDER_MAC_AUTHENTICATED 0x0800000000000000ULL + /** * Packet characteristics flag: TCP left-most reserved bit */ diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp index 8305c489..6fc59ad1 100644 --- a/node/CertificateOfOwnership.cpp +++ b/node/CertificateOfOwnership.cpp @@ -43,4 +43,21 @@ int CertificateOfOwnership::verify(const RuntimeEnvironment *RR) const } } +bool CertificateOfOwnership::_owns(const CertificateOfOwnership::Thing &t,const void *v,unsigned int l) const +{ + for(unsigned int i=0,j=_thingCount;i(v)[k] != _thingValues[i][k]) + break; + ++k; + } + if (k == l) + return true; + } + } + return false; +} + } // namespace ZeroTier diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 69b26aec..7e71c9b2 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -84,37 +84,20 @@ public: inline const Address &issuedTo() const { return _issuedTo; } - inline bool owns(const Thing &t,const void *v,unsigned int l) - { - for(unsigned int i=0,j=_thingCount;i(v)[k] != _thingValues[i][k]) - break; - ++k; - } - if (k == l) - return true; - } - } - return false; - } - - inline bool owns(const InetAddress &ip) + inline bool owns(const InetAddress &ip) const { if (ip.ss_family == AF_INET) - return this->owns(THING_IPV4_ADDRESS,&(reinterpret_cast(&ip)->sin_addr.s_addr),4); + return this->_owns(THING_IPV4_ADDRESS,&(reinterpret_cast(&ip)->sin_addr.s_addr),4); if (ip.ss_family == AF_INET6) - return this->owns(THING_IPV6_ADDRESS,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + return this->_owns(THING_IPV6_ADDRESS,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); return false; } - inline bool owns(const MAC &mac) + inline bool owns(const MAC &mac) const { uint8_t tmp[6]; mac.copyTo(tmp,6); - return this->owns(THING_MAC_ADDRESS,tmp,6); + return this->_owns(THING_MAC_ADDRESS,tmp,6); } inline void addThing(const InetAddress &ip) @@ -234,6 +217,8 @@ public: inline bool operator!=(const CertificateOfOwnership &coo) const { return (memcmp(this,&coo,sizeof(CertificateOfOwnership)) != 0); } private: + bool _owns(const Thing &t,const void *v,unsigned int l) const; + uint64_t _networkId; uint64_t _ts; uint64_t _flags; diff --git a/node/Membership.cpp b/node/Membership.cpp index 1eacb93d..5facbe24 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -117,16 +117,10 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now } } -const Capability *Membership::getCapability(const NetworkConfig &nconf,const uint32_t id) const -{ - const _RemoteCredential *const *c = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)id,_RemoteCredentialComp()); - return ( ((c != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*c)->id == (uint64_t)id)) ? ((((*c)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*c)->credential,**c))) ? &((*c)->credential) : (const Capability *)0) : (const Capability *)0); -} - const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) const { const _RemoteCredential *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialComp()); - return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,(*t)->credential,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); + return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) @@ -165,7 +159,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,tag,*have)) || (have->credential.timestamp() > tag.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > tag.timestamp()) ) { TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_REJECTED; } @@ -195,7 +189,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,cap,*have)) || (have->credential.timestamp() > cap.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > cap.timestamp()) ) { TRACE("addCredential(Capability) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; } @@ -252,7 +246,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; if (have) { - if ( (!_isCredentialTimestampValid(nconf,coo,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { + if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; } @@ -298,7 +292,7 @@ Membership::_RemoteCredential *Membership::_newTag(const uint64_t id) t->credential = Tag(); } - std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS])); + std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialComp()); return t; } @@ -323,7 +317,7 @@ Membership::_RemoteCredential *Membership::_newCapability(const uint c->credential = Capability(); } - std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES])); + std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialComp()); return c; } @@ -348,7 +342,7 @@ Membership::_RemoteCredential *Membership::_newCoo(const c->credential = CertificateOfOwnership(); } - std::sort(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP])); + std::sort(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),_RemoteCredentialComp()); return c; } diff --git a/node/Membership.hpp b/node/Membership.hpp index 4e9d7769..a7794328 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -99,7 +99,7 @@ public: for(;;) { if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { const Capability *tmp = &((*_i)->credential); - if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { + if (_m->_isCredentialTimestampValid(*_c,**_i)) { ++_i; return tmp; } else ++_i; @@ -132,7 +132,7 @@ public: for(;;) { if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { const Tag *tmp = &((*_i)->credential); - if (_m->_isCredentialTimestampValid(*_c,*tmp,**_i)) { + if (_m->_isCredentialTimestampValid(*_c,**_i)) { ++_i; return tmp; } else ++_i; @@ -197,11 +197,24 @@ public: } /** - * @param nconf Network configuration - * @param id Capablity ID - * @return Pointer to capability or NULL if not found + * Check whether the peer represented by this Membership owns a given resource + * + * @tparam Type of resource: InetAddress or MAC + * @param nconf Our network config + * @param r Resource to check + * @return True if this peer has a certificate of ownership for the given resource */ - const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const; + template + inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) const + { + for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) + break; + if ((_isCredentialTimestampValid(nconf,*_remoteCoos[i]))&&(_remoteCoos[i]->credential.owns(r))) + return true; + } + return false; + } /** * @param nconf Network configuration @@ -244,11 +257,13 @@ private: bool _revokeTag(const Revocation &rev,const uint64_t now); bool _revokeCoo(const Revocation &rev,const uint64_t now); - template - inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &cred,const CS &state) const + template + inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const _RemoteCredential &remoteCredential) const { - const uint64_t ts = cred.timestamp(); - return ( (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) && (ts > state.revocationThreshold) ); + if (!remoteCredential.lastReceived) + return false; + const uint64_t ts = remoteCredential.credential.timestamp(); + return ( (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) && (ts > remoteCredential.revocationThreshold) ); } // Last time we pushed MULTICAST_LIKE(s) diff --git a/node/Network.cpp b/node/Network.cpp index 290ceaf9..50df58bb 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -299,6 +299,7 @@ static _doZtFilterResult _doZtFilter( // If this was not an ACTION evaluate next MATCH and update thisSetMatches with (AND [result]) uint8_t thisRuleMatches = 0; + uint64_t ownershipVerificationMask = 1; // this magic value means it hasn't been computed yet -- this is done lazily the first time it's needed switch(rt) { case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); @@ -507,6 +508,30 @@ static _doZtFilterResult _doZtFilter( uint64_t cf = (inbound) ? ZT_RULE_PACKET_CHARACTERISTICS_INBOUND : 0ULL; if (macDest.isMulticast()) cf |= ZT_RULE_PACKET_CHARACTERISTICS_MULTICAST; if (macDest.isBroadcast()) cf |= ZT_RULE_PACKET_CHARACTERISTICS_BROADCAST; + if (ownershipVerificationMask == 1) { + ownershipVerificationMask = 0; + InetAddress src; + if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { + src.set((const void *)(frameData + 12),4,0); + } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { + src.set((const void *)(frameData + 8),16,0); + } + if (inbound) { + if (membership) { + if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) + ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED; + if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) + ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_MAC_AUTHENTICATED; + } + } else { + for(unsigned int i=0;i= 20)&&(frameData[9] == 0x06)) { const unsigned int headerLen = 4 * (frameData[0] & 0xf); cf |= (uint64_t)frameData[headerLen + 13]; diff --git a/rule-compiler/package.json b/rule-compiler/package.json index 16f7a23f..f0e747a2 100644 --- a/rule-compiler/package.json +++ b/rule-compiler/package.json @@ -1,6 +1,6 @@ { "name": "zerotier-rule-compiler", - "version": "1.1.17-1", + "version": "1.1.17-2", "description": "ZeroTier Rule Script Compiler", "main": "cli.js", "scripts": { diff --git a/rule-compiler/rule-compiler.js b/rule-compiler/rule-compiler.js index d927e512..be4e5c64 100644 --- a/rule-compiler/rule-compiler.js +++ b/rule-compiler/rule-compiler.js @@ -5,6 +5,8 @@ const CHARACTERISTIC_BITS = { 'inbound': 63, 'multicast': 62, 'broadcast': 61, + 'ipauth': 60, + 'macauth': 59, 'tcp_fin': 0, 'tcp_syn': 1, 'tcp_rst': 2, -- cgit v1.2.3 From 9d7ff26f254fce03d0fd165df61cb8465ac21220 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 23 Feb 2017 14:27:31 -0800 Subject: Helps if you actually add the ipauth mask to the characteristics mask. --- node/Network.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 50df58bb..f227d036 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -532,6 +532,7 @@ static _doZtFilterResult _doZtFilter( } } } + cf |= ownershipVerificationMask; if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)&&(frameData[9] == 0x06)) { const unsigned int headerLen = 4 * (frameData[0] & 0xf); cf |= (uint64_t)frameData[headerLen + 13]; -- cgit v1.2.3 From 4436824faf726014bf3aa47f6c8d2748ca793ba2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 27 Feb 2017 17:51:58 -0800 Subject: ipauth characteristic now works with ARP --- node/Membership.cpp | 1 - node/Network.cpp | 2 ++ node/OutboundMulticast.cpp | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index d9fa5945..a60b86be 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -240,7 +240,6 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); diff --git a/node/Network.cpp b/node/Network.cpp index f227d036..645ae67c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -515,6 +515,8 @@ static _doZtFilterResult _doZtFilter( src.set((const void *)(frameData + 12),4,0); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { src.set((const void *)(frameData + 8),16,0); + } else if ((etherType == ZT_ETHERTYPE_ARP)&&(frameLen >= 28)) { + src.set((const void *)(frameData + 14),4,0); } if (inbound) { if (membership) { diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 2f6bf986..36dc41f4 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -88,7 +88,7 @@ void OutboundMulticast::init( void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) { const SharedPtr nw(RR->node->network(_nwid)); - Address toAddr2(toAddr); + const Address toAddr2(toAddr); if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); -- cgit v1.2.3 From 31bece7fa076cf1c5f21743a9a936ade54c0fe1a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 28 Feb 2017 07:43:40 -0800 Subject: Add ipauth handling of IPv6 NDP neighbor solicitations and advertisements. IPv6 works well now with ipauth. --- node/Network.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 645ae67c..aad6e716 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -514,7 +514,22 @@ static _doZtFilterResult _doZtFilter( if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { src.set((const void *)(frameData + 12),4,0); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { - src.set((const void *)(frameData + 8),16,0); + // IPv6 NDP requires special handling, since the src and dest IPs in the packet are empty or link-local. + unsigned int pos = 0,proto = 0; + if ( (frameLen >= (40 + 8 + 16)) && (frameData[6] == 0x3a) && ((frameData[40] == 0x87)||(frameData[40] == 0x88)) ) { + if (frameData[40] == 0x87) { + // Neighbor solicitations contain no reliable source address, so we implement a small + // hack by considering them authenticated. Otherwise you would pretty much have to do + // this manually in the rule set for IPv6 to work at all. + ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED; + } else { + // Neighbor advertisements on the other hand can absolutely be authenticated. + src.set((const void *)(frameData + 40 + 8),16,0); + } + } else { + // Other IPv6 packets can be handled normally + src.set((const void *)(frameData + 8),16,0); + } } else if ((etherType == ZT_ETHERTYPE_ARP)&&(frameLen >= 28)) { src.set((const void *)(frameData + 14),4,0); } -- cgit v1.2.3 From 2b10a982e9c9aad4a6ed9b10f5d7bda93a55ffcf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 28 Feb 2017 09:22:10 -0800 Subject: Match on tag sender equals or tag recipient equals. --- controller/EmbeddedNetworkController.cpp | 36 ++++++++++++++++++++---------- include/ZeroTierOne.h | 2 ++ node/Capability.hpp | 8 ++++--- node/Network.cpp | 38 +++++++++++++++++++++++++++++++- rule-compiler/package.json | 2 +- rule-compiler/rule-compiler.js | 14 +++++++++--- 6 files changed, 80 insertions(+), 20 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 78a9b7c7..0e57fc14 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -218,6 +218,16 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; break; + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + r["type"] = "MATCH_TAG_SENDER"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: + r["type"] = "MATCH_TAG_RECEIVER"; + r["id"] = rule.v.tag.id; + r["value"] = rule.v.tag.value; + break; default: break; } @@ -245,6 +255,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) if (OSUtils::jsonBool(r["or"],false)) rule.t |= 0x40; + bool tag = false; if (t == "ACTION_DROP") { rule.t |= ZT_NETWORK_RULE_ACTION_DROP; return true; @@ -388,26 +399,27 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_TAGS_DIFFERENCE") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_BITWISE_AND") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_BITWISE_OR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_BITWISE_XOR") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); - return true; + tag = true; } else if (t == "MATCH_TAGS_EQUAL") { rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_EQUAL; + tag = true; + } else if (t == "MATCH_TAG_SENDER") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAG_SENDER; + tag = true; + } else if (t == "MATCH_TAG_RECEIVER") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAG_RECEIVER; + tag = true; + } + if (tag) { rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); return true; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8a13c117..2c141f47 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -604,6 +604,8 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR = 46, ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR = 47, ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 48, + ZT_NETWORK_RULE_MATCH_TAG_SENDER = 49, + ZT_NETWORK_RULE_MATCH_TAG_RECEIVER = 50, /** * Maximum ID allowed for a MATCH entry in the rules table diff --git a/node/Capability.hpp b/node/Capability.hpp index d884625e..1ad6ea42 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -167,9 +167,6 @@ public: // rules to be ignored but still parsed. b.append((uint8_t)rules[i].t); switch((ZT_VirtualNetworkRuleType)(rules[i].t & 0x3f)) { - //case ZT_NETWORK_RULE_ACTION_DROP: - //case ZT_NETWORK_RULE_ACTION_ACCEPT: - //case ZT_NETWORK_RULE_ACTION_DEBUG_LOG: default: b.append((uint8_t)0); break; @@ -258,6 +255,9 @@ public: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: b.append((uint8_t)8); b.append((uint32_t)rules[i].v.tag.id); b.append((uint32_t)rules[i].v.tag.value); @@ -345,6 +345,8 @@ public: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: case ZT_NETWORK_RULE_MATCH_TAGS_EQUAL: + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: rules[ruleCount].v.tag.id = b.template at(p); rules[ruleCount].v.tag.value = b.template at(p + 4); break; diff --git a/node/Network.cpp b/node/Network.cpp index aad6e716..dc976f03 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -515,7 +515,6 @@ static _doZtFilterResult _doZtFilter( src.set((const void *)(frameData + 12),4,0); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { // IPv6 NDP requires special handling, since the src and dest IPs in the packet are empty or link-local. - unsigned int pos = 0,proto = 0; if ( (frameLen >= (40 + 8 + 16)) && (frameData[6] == 0x3a) && ((frameData[40] == 0x87)||(frameData[40] == 0x88)) ) { if (frameData[40] == 0x87) { // Neighbor solicitations contain no reliable source address, so we implement a small @@ -609,6 +608,11 @@ static _doZtFilterResult _doZtFilter( thisRuleMatches = 0; FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { + // Outbound side is not strict since if we have to match both tags and + // we are sending a first packet to a recipient, we probably do not know + // about their tags yet. They will filter on inbound and we will filter + // once we get their tag. If we are a tee/redirect target we are also + // not strict since we likely do not have these tags. thisRuleMatches = 1; FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side and TEE/REDIRECT targets are not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } @@ -618,6 +622,38 @@ static _doZtFilterResult _doZtFilter( FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } break; + case ZT_NETWORK_RULE_MATCH_TAG_SENDER: + case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: { + if (superAccept) { + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c we are a TEE/REDIRECT target -> 1",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); + } else if ( ((rt == ZT_NETWORK_RULE_MATCH_TAG_SENDER)&&(inbound)) || ((rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER)&&(!inbound)) ) { + const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); + if (remoteTag) { + thisRuleMatches = (uint8_t)(remoteTag->value() == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + } else { + if (rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER) { + // If we are checking the receiver and this is an outbound packet, we + // can't be strict since we may not yet know the receiver's tag. + thisRuleMatches = 1; + FILTER_TRACE("%u %s %c (inbound) remote tag %u not found -> 1 (outbound receiver match is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c (inbound) remote tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } + } + } else { // sender and outbound or receiver and inbound + const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); + if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { + thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value); + FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + } else { + thisRuleMatches = 0; + FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); + } + } + } break; // The result of an unsupported MATCH is configurable at the network // level via a flag. diff --git a/rule-compiler/package.json b/rule-compiler/package.json index f0e747a2..451295a4 100644 --- a/rule-compiler/package.json +++ b/rule-compiler/package.json @@ -1,6 +1,6 @@ { "name": "zerotier-rule-compiler", - "version": "1.1.17-2", + "version": "1.1.17-3", "description": "ZeroTier Rule Script Compiler", "main": "cli.js", "scripts": { diff --git a/rule-compiler/rule-compiler.js b/rule-compiler/rule-compiler.js index 7445d39f..feb30f90 100644 --- a/rule-compiler/rule-compiler.js +++ b/rule-compiler/rule-compiler.js @@ -105,6 +105,8 @@ const RESERVED_WORDS = { 'txor': true, 'tdiff': true, 'teq': true, + 'tseq': true, + 'treq': true, 'type': true, 'enum': true, @@ -152,7 +154,9 @@ const KEYWORD_TO_API_MAP = { 'tor': 'MATCH_TAGS_BITWISE_OR', 'txor': 'MATCH_TAGS_BITWISE_XOR', 'tdiff': 'MATCH_TAGS_DIFFERENCE', - 'teq': 'MATCH_TAGS_EQUAL' + 'teq': 'MATCH_TAGS_EQUAL', + 'tseq': 'MATCH_TAG_SENDER', + 'treq': 'MATCH_TAG_RECEIVER' }; // Number of args for each match @@ -179,7 +183,9 @@ const MATCH_ARG_COUNTS = { 'tor': 2, 'txor': 2, 'tdiff': 2, - 'teq': 2 + 'teq': 2, + 'tseq': 2, + 'treq': 2 }; // Regex of all alphanumeric characters in Unicode @@ -477,7 +483,9 @@ function _renderMatches(mtree,rules,macros,caps,tags,params) case 'tor': case 'txor': case 'tdiff': - case 'teq': { + case 'teq': + case 'tseq': + case 'treq': { let tag = tags[args[0][0]]; let tagId = -1; let tagValue = -1; -- cgit v1.2.3 From 127bcb02ffd09b522678c7e50aae21a1ecd87e4e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 09:41:37 -0800 Subject: Save space in expecting-reply-to tracking. --- node/Node.hpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Node.hpp b/node/Node.hpp index a1d4b719..39cf2601 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -219,25 +219,34 @@ public: /** * Register that we are expecting a reply to a packet ID * + * This only uses the most significant bits of the packet ID, both to save space + * and to avoid using the higher bits that can be modified during armor() to + * mask against the packet send counter used for QoS detection. + * * @param packetId Packet ID to expect reply to */ inline void expectReplyTo(const uint64_t packetId) { const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); - _expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = packetId; + _expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = (uint32_t)(packetId >> 32); } /** * Check whether a given packet ID is something we are expecting a reply to * + * This only uses the most significant bits of the packet ID, both to save space + * and to avoid using the higher bits that can be modified during armor() to + * mask against the packet send counter used for QoS detection. + * * @param packetId Packet ID to check * @return True if we're expecting a reply */ inline bool expectingReplyTo(const uint64_t packetId) const { const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); + const uint32_t pid = (uint32_t)(packetId >> 32); for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) { - if (_expectingRepliesTo[bucket][i] == packetId) + if (_expectingRepliesTo[bucket][i] == pid) return true; } return false; @@ -281,9 +290,9 @@ private: // For tracking packet IDs to filter out OK/ERROR replies to packets we did not send uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1]; - uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; + uint32_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; - // Time of last identity verification indexed by InetAddress.rateGateHash() + // Time of last identity verification indexed by InetAddress.rateGateHash() -- used in IncomingPacket::_doHELLO() via rateGateIdentityVerification() uint64_t _lastIdentityVerification[16384]; std::vector< std::pair< uint64_t, SharedPtr > > _networks; -- cgit v1.2.3 From 2bf9145ae65385bf968542619ffcf204cf6241d8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 10:22:57 -0800 Subject: Outgoing side of packet counter for link quality reporting. Also some cleanup and a cluster mode build fix. --- node/Buffer.hpp | 44 +- node/Cluster.cpp | 8 +- node/IncomingPacket.cpp | 26 +- node/Node.cpp | 6 +- node/Packet.cpp | 1942 +++++++++++++++++++++++------------------------ node/Packet.hpp | 25 +- node/Path.hpp | 10 + node/Peer.cpp | 24 +- node/Peer.hpp | 6 +- node/Switch.cpp | 8 +- 10 files changed, 1053 insertions(+), 1046 deletions(-) (limited to 'node') diff --git a/node/Buffer.hpp b/node/Buffer.hpp index 1a478894..37f39e7b 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -79,8 +79,7 @@ public: inline const_reverse_iterator rbegin() const { return const_reverse_iterator(begin()); } inline const_reverse_iterator rend() const { return const_reverse_iterator(end()); } - Buffer() - throw() : + Buffer() : _l(0) { } @@ -419,87 +418,70 @@ public: /** * Set buffer data length to zero */ - inline void clear() - throw() - { - _l = 0; - } + inline void clear() { _l = 0; } /** * Zero buffer up to size() */ - inline void zero() - throw() - { - memset(_b,0,_l); - } + inline void zero() { memset(_b,0,_l); } /** * Zero unused capacity area */ - inline void zeroUnused() - throw() - { - memset(_b + _l,0,C - _l); - } + inline void zeroUnused() { memset(_b + _l,0,C - _l); } /** * Unconditionally and securely zero buffer's underlying memory */ - inline void burn() - throw() - { - Utils::burn(_b,sizeof(_b)); - } + inline void burn() { Utils::burn(_b,sizeof(_b)); } /** * @return Constant pointer to data in buffer */ - inline const void *data() const throw() { return _b; } + inline const void *data() const { return _b; } + + /** + * @return Non-constant pointer to data in buffer + */ + inline void *unsafeData() { return _b; } /** * @return Size of data in buffer */ - inline unsigned int size() const throw() { return _l; } + inline unsigned int size() const { return _l; } /** * @return Capacity of buffer */ - inline unsigned int capacity() const throw() { return C; } + inline unsigned int capacity() const { return C; } template inline bool operator==(const Buffer &b) const - throw() { return ((_l == b._l)&&(!memcmp(_b,b._b,_l))); } template inline bool operator!=(const Buffer &b) const - throw() { return ((_l != b._l)||(memcmp(_b,b._b,_l))); } template inline bool operator<(const Buffer &b) const - throw() { return (memcmp(_b,b._b,std::min(_l,b._l)) < 0); } template inline bool operator>(const Buffer &b) const - throw() { return (b < *this); } template inline bool operator<=(const Buffer &b) const - throw() { return !(b < *this); } template inline bool operator>=(const Buffer &b) const - throw() { return !(*this < b); } diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 00122402..52e03ffe 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -255,7 +255,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") char polykey[ZT_POLY1305_KEY_LEN]; memset(polykey,0,sizeof(polykey)); - s20.encrypt12(polykey,polykey,sizeof(polykey)); + s20.crypt12(polykey,polykey,sizeof(polykey)); // Compute 16-byte MAC char mac[ZT_POLY1305_MAC_LEN]; @@ -267,7 +267,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) // Decrypt! dmsg.setSize(len - 24); - s20.decrypt12(reinterpret_cast(msg) + 24,const_cast(dmsg.data()),dmsg.size()); + s20.crypt12(reinterpret_cast(msg) + 24,const_cast(dmsg.data()),dmsg.size()); } if (dmsg.size() < 4) @@ -954,10 +954,10 @@ void Cluster::_flush(uint16_t memberId) // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") char polykey[ZT_POLY1305_KEY_LEN]; memset(polykey,0,sizeof(polykey)); - s20.encrypt12(polykey,polykey,sizeof(polykey)); + s20.crypt12(polykey,polykey,sizeof(polykey)); // Encrypt m.q in place - s20.encrypt12(reinterpret_cast(m.q.data()) + 24,const_cast(reinterpret_cast(m.q.data())) + 24,m.q.size() - 24); + s20.crypt12(reinterpret_cast(m.q.data()) + 24,const_cast(reinterpret_cast(m.q.data())) + 24,m.q.size() - 24); // Add MAC for authentication (encrypt-then-MAC) char mac[ZT_POLY1305_MAC_LEN]; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index b5b2bcb3..85b06d50 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -243,7 +243,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut outp.append((uint8_t)Packet::VERB_HELLO); outp.append((uint64_t)pid); outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION); - outp.armor(key,true); + outp.armor(key,true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } else { TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); @@ -405,7 +405,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut RR->topology->appendCertificateOfRepresentation(outp); outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version @@ -584,7 +584,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr } if (count > 0) { - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } @@ -610,7 +610,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); if (RR->node->shouldUsePathForZeroTierTraffic(with,_path->localAddress(),atAddr)) { RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now(),false); + rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now(),false,0); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -732,7 +732,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } @@ -762,7 +762,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr outp.append((uint64_t)pid); if (size() > ZT_PACKET_IDX_PAYLOAD) outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); @@ -957,7 +957,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append(requestPacketId); outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); outp.append(nwid); - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } @@ -984,7 +984,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared outp.append((uint64_t)packetId()); outp.append((uint64_t)network->id()); outp.append((uint64_t)configUpdateId); - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } } @@ -1033,7 +1033,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar outp.append((uint32_t)mg.adi()); const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); if (gatheredLocally > 0) { - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } @@ -1140,7 +1140,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share outp.append((uint32_t)to.adi()); outp.append((unsigned char)0x02); // flag 0x02 = contains gather results if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) { - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),RR->node->now()); } } @@ -1198,7 +1198,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now,false); + peer->attemptToContactAt(InetAddress(),a,now,false,0); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1217,7 +1217,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now,false); + peer->attemptToContactAt(InetAddress(),a,now,false,0); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1447,7 +1447,7 @@ void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,cons outp.append(packetId()); outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); outp.append(nwid); - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,outp.data(),outp.size(),now); } } diff --git a/node/Node.cpp b/node/Node.cpp index 6dc89387..35940d27 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -180,7 +180,7 @@ public: for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET) { - p->sendHELLO(InetAddress(),addr,_now); + p->sendHELLO(InetAddress(),addr,_now,0); contacted = true; break; } @@ -190,7 +190,7 @@ public: for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET6) { - p->sendHELLO(InetAddress(),addr,_now); + p->sendHELLO(InetAddress(),addr,_now,0); contacted = true; break; } @@ -200,7 +200,7 @@ public: if ((!contacted)&&(_bestCurrentUpstream)) { const SharedPtr up(_bestCurrentUpstream->getBestPath(_now,true)); if (up) - p->sendHELLO(up->localAddress(),up->address(),_now); + p->sendHELLO(up->localAddress(),up->address(),_now,up->nextOutgoingCounter()); } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); diff --git a/node/Packet.cpp b/node/Packet.cpp index 790f4b09..82a5d7ea 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -65,9 +65,9 @@ namespace { modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. @@ -85,8 +85,8 @@ namespace { OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 */ /* --- Dependency --- */ @@ -101,9 +101,9 @@ namespace { The LZ4 compression library provides in-memory compression and decompression functions. Compression can be done in: - - a single step (described as Simple Functions) - - a single step, reusing a context (described in Advanced Functions) - - unbounded multiple steps (described as Streaming compression) + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) lz4.h provides block compression functions. It gives full buffer control to user. Decompressing an lz4-compressed block also requires metadata (such as compressed size). @@ -164,28 +164,28 @@ namespace { * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails */ //LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); /*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. */ LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); @@ -198,33 +198,33 @@ LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compress /*! LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) - inputSize : max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) */ LZ4LIB_API int LZ4_compressBound(int inputSize); /*! LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. - The larger the acceleration value, the faster the algorithm, but also the lesser the compression. - It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. + Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. */ LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); /*! LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. + Same compression function, just using an externally allocated memory space to store compression state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. */ //LZ4LIB_API int LZ4_sizeofState(void); LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); @@ -232,42 +232,42 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char /*! LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails */ //LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); /*! LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ4_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). */ //LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); /*! LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets */ //LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); @@ -336,20 +336,20 @@ typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defin /*! LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() */ //LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); //LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); @@ -382,37 +382,37 @@ LZ4_decompress_*_continue() : //#include typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ - uint32_t dictSize; + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + uint32_t dictSize; } LZ4_stream_t_internal; typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; } LZ4_streamDecode_t_internal; #else typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + unsigned int dictSize; } LZ4_stream_t_internal; typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; } LZ4_streamDecode_t_internal; #endif @@ -428,8 +428,8 @@ typedef struct { #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; - LZ4_stream_t_internal internal_donotuse; + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ @@ -444,8 +444,8 @@ union LZ4_stream_u { #define LZ4_STREAMDECODESIZE_U64 4 #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; - LZ4_streamDecode_t_internal internal_donotuse; + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ /* lz4.c ------------------------------------------------------------------ */ @@ -460,9 +460,9 @@ union LZ4_streamDecode_u { modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. @@ -480,8 +480,8 @@ union LZ4_streamDecode_u { OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 */ @@ -617,8 +617,8 @@ typedef uintptr_t reg_t; **************************************/ static unsigned LZ4_isLittleEndian(void) { - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; } #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) @@ -648,27 +648,27 @@ static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = val static U16 LZ4_read16(const void* memPtr) { - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; } static U32 LZ4_read32(const void* memPtr) { - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; } static reg_t LZ4_read_ARCH(const void* memPtr) { - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; + reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; } static void LZ4_write16(void* memPtr, U16 value) { - memcpy(memPtr, &value, sizeof(value)); + memcpy(memPtr, &value, sizeof(value)); } static void LZ4_write32(void* memPtr, U32 value) { - memcpy(memPtr, &value, sizeof(value)); + memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ @@ -676,38 +676,38 @@ static void LZ4_write32(void* memPtr, U32 value) static U16 LZ4_readLE16(const void* memPtr) { - if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); - } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); - } + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } } static void LZ4_writeLE16(void* memPtr, U16 value) { - if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - } + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } } static void LZ4_copy8(void* dst, const void* src) { - memcpy(dst,src,8); + memcpy(dst,src,8); } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); + return (__builtin_ctzll((U64)val) >> 3); # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif - } else /* 32 bits */ { + } else /* 32 bits */ { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); + return (__builtin_ctz((U32)val) >> 3); # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif - } - } else /* Big Endian CPU */ { - if (sizeof(val)==8) { + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); + return (__builtin_clzll((U64)val) >> 3); # else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; # endif - } else /* 32 bits */ { + } else /* 32 bits */ { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); + return (__builtin_clz((U32)val) >> 3); # else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; # endif - } - } + } + } } #define STEPSIZE sizeof(reg_t) static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { - const BYTE* const pStart = pIn; - - while (likely(pIn> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; - const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + static const U64 prime5bytes = 889523592379ULL; + static const U64 prime8bytes = 11400714785074694791ULL; + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + else + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) { - if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); - return LZ4_hash4(LZ4_read32(p), tableType); + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { - switch (tableType) - { - case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } } FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { - U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } /** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ + inlined, to ensure branches are decided at compilation time */ FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) { - const BYTE* ip = (const BYTE*) source; - const BYTE* base; - const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; - - U32 forwardH; - - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; - } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizehashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } - - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength<dictSize; + const BYTE* const dictionary = cctx->dictionary; + const BYTE* const dictEnd = dictionary + cctx->dictSize; + const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source - cctx->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSizehashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + ptrdiff_t refDelta = 0; + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; + if (litLength >= RUN_MASK) { + int len = (int)litLength-RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } - - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } + /* Encode Offset */ + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + + /* Encode MatchLength */ + { unsigned matchCode; + + if ((dict==usingExtDict) && (lowLimit==dictionary)) { + const BYTE* limit; + match += refDelta; + limit = ip + (dictEnd-match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchCode; + if (ip==limit) { + unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchCode; + } + + if ( outputLimited && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) + return 0; + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: - /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun< (U32)maxOutputSize) ) + return 0; + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun<internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + LZ4_resetStream((LZ4_stream_t*)state); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } else { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } } int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { #if (HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; + LZ4_stream_t ctx; + void* const ctxPtr = &ctx; #endif - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (HEAPMODE) - FREEMEM(ctxPtr); + FREEMEM(ctxPtr); #endif - return result; + return result; } #if 0 int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) { - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); } /* hidden debug function */ /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); + LZ4_stream_t ctx; + LZ4_resetStream(&ctx); - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); } #endif @@ -1173,189 +1173,189 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m #if 0 static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) + LZ4_stream_t_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) { - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtrhashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = 1 << LZ4_skipTrigger; + + do { + U32 h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + /* Encode Literal length */ + { unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } + /* Encode Offset */ + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + + /* Encode MatchLength */ + { size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + if (op + ((matchLength+240)/255) > oMaxMatch) { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize< oend) { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } + LZ4_resetStream(state); + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } else { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); + } } int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; + LZ4_stream_t ctxBody; + LZ4_stream_t* ctx = &ctxBody; #endif - int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); #if (HEAPMODE) - FREEMEM(ctx); + FREEMEM(ctx); #endif - return result; + return result; } #endif @@ -1366,23 +1366,23 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe #if 0 LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; } #endif void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } #if 0 int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { - FREEMEM(LZ4_stream); - return (0); + FREEMEM(LZ4_stream); + return (0); } #endif @@ -1390,117 +1390,117 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) #define HASH_UNIT sizeof(reg_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; - - while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); - p+=3; - } - - return dict->dictSize; + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); + + if (dictSize < (int)HASH_UNIT) { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + LZ4_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } + + return dict->dictSize; } static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } + if ((LZ4_dict->currentOffset > 0x80000000) || + ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } } int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - /* external dictionary mode */ - { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } } /* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT(streamPtr, smallest); - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; - return result; + return result; } /*! LZ4_saveDict() : @@ -1512,18 +1512,18 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; - return dictSize; + return dictSize; } #endif @@ -1538,181 +1538,181 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) * in order to remove useless branches during compilation optimization. */ FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) { - /* Local Variables */ - const BYTE* ip = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; - - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; - - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - - /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - - /* Main Loop : decode sequences */ - while (1) { - size_t length; - const BYTE* match; - size_t offset; - - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); - } - op=cpy; /* correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-source); /* Nb of input bytes read */ - - /* Overflow error detected */ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; + const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + /* Main Loop : decode sequences */ + while (1) { + size_t length; + const BYTE* match; + size_t offset; + + /* get literal length */ + unsigned const token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) { + unsigned s; + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + memcpy(op+4, match, 4); + match -= dec64; + } else { LZ4_copy8(op, match); match+=8; } + op += 8; + + if (unlikely(cpy>oend-12)) { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op16) LZ4_wildCopy(op+8, match+8, cpy); + } + op=cpy; /* correction */ + } + + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-source); /* Nb of input bytes read */ + + /* Overflow error detected */ _output_error: - return (int) (-(((const char*)ip)-source))-1; + return (int) (-(((const char*)ip)-source))-1; } int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); } #if 0 int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); } int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); } #endif @@ -1726,14 +1726,14 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); - return lz4s; + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - FREEMEM(LZ4_stream); - return 0; + FREEMEM(LZ4_stream); + return 0; } /*! @@ -1745,107 +1745,107 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; } /* *_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; } int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; } /* Advanced decoding functions : *_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters */ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) { - if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); } /* debug function */ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } #endif @@ -1878,41 +1878,41 @@ int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) { - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; } int LZ4_resetStreamState(void* state, char* inputBuffer) { - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); - return 0; + if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + return 0; } void* LZ4_create (char* inputBuffer) { - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); + return lz4ds; } char* LZ4_slideInputBuffer (void* LZ4_Data) { - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; + int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); } /* Obsolete streaming decompression functions */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } #endif @@ -1928,7 +1928,6 @@ const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 #ifdef ZT_TRACE const char *Packet::verbString(Verb v) - throw() { switch(v) { case VERB_NOP: return "NOP"; @@ -1955,7 +1954,6 @@ const char *Packet::verbString(Verb v) } const char *Packet::errorString(ErrorCode e) - throw() { switch(e) { case ERROR_NONE: return "NONE"; @@ -1973,54 +1971,56 @@ const char *Packet::errorString(ErrorCode e) #endif // ZT_TRACE -void Packet::armor(const void *key,bool encryptPayload) +void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) { - unsigned char mangledKey[32]; - unsigned char macKey[32]; - unsigned char mac[16]; - const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; - unsigned char *const payload = field(ZT_PACKET_IDX_VERB,payloadLen); + uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t *const data = reinterpret_cast(unsafeData()); + + // Mask least significant 3 bits of packet ID with counter to embed packet send counter for QoS use + data[7] = (data[7] & 0xf8) | ((uint8_t)counter & 0x07); // Set flag now, since it affects key mangle function setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE); _salsa20MangleKey((const unsigned char *)key,mangledKey); - Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)/*,ZT_PROTO_SALSA20_ROUNDS*/); + Salsa20 s20(mangledKey,256,data + ZT_PACKET_IDX_IV); // MAC key is always the first 32 bytes of the Salsa20 key stream // This is the same construction DJB's NaCl library uses s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint8_t *const payload = data + ZT_PACKET_IDX_VERB; + const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; if (encryptPayload) s20.crypt12(payload,payload,payloadLen); - Poly1305::compute(mac,payload,payloadLen,macKey); - memcpy(field(ZT_PACKET_IDX_MAC,8),mac,8); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); } bool Packet::dearmor(const void *key) { - unsigned char mangledKey[32]; - unsigned char macKey[32]; - unsigned char mac[16]; + uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t *const data = reinterpret_cast(unsafeData()); const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; - unsigned char *const payload = field(ZT_PACKET_IDX_VERB,payloadLen); - unsigned int cs = cipher(); + unsigned char *const payload = data + ZT_PACKET_IDX_VERB; + const unsigned int cs = cipher(); if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); - Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)); + Salsa20 s20(mangledKey,256,data + ZT_PACKET_IDX_IV); s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); Poly1305::compute(mac,payload,payloadLen,macKey); - if (!Utils::secureEq(mac,field(ZT_PACKET_IDX_MAC,8),8)) - return false; + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) s20.crypt12(payload,payload,payloadLen); return true; - } else return false; // unrecognized cipher suite + } else { + return false; // unrecognized cipher suite + } } void Packet::cryptField(const void *key,unsigned int start,unsigned int len) @@ -2028,13 +2028,13 @@ void Packet::cryptField(const void *key,unsigned int start,unsigned int len) unsigned char mangledKey[32]; unsigned char macKey[32]; _salsa20MangleKey((const unsigned char *)key,mangledKey); - mangledKey[0] ^= 0x7f; - mangledKey[1] ^= ((start >> 8) & 0xff); - mangledKey[2] ^= (start & 0xff); // slightly alter key for this use case as an added guard against key stream reuse + mangledKey[0] ^= 0x7f; + mangledKey[1] ^= ((start >> 8) & 0xff); + mangledKey[2] ^= (start & 0xff); // slightly alter key for this use case as an added guard against key stream reuse Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)); s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); // discard the first 32 bytes of key stream (the ones use for MAC in armor()) as a precaution - unsigned char *const ptr = field(start,len); - s20.crypt12(ptr,ptr,len); + unsigned char *const ptr = field(start,len); + s20.crypt12(ptr,ptr,len); } bool Packet::compress() diff --git a/node/Packet.hpp b/node/Packet.hpp index 6482356a..2017ce8e 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -351,7 +351,7 @@ namespace ZeroTier { * ZeroTier packet * * Packet format: - * <[8] 64-bit random packet ID and crypto initialization vector> + * <[8] 64-bit packet ID / crypto IV / packet counter> * <[5] destination ZT address> * <[5] source ZT address> * <[1] flags/cipher/hops> @@ -362,6 +362,14 @@ namespace ZeroTier { * * Packets smaller than 28 bytes are invalid and silently discarded. * + * The 64-bit packet ID is a strongly random value used as a crypto IV. + * Its least significant 3 bits are also used as a monotonically increasing + * (and looping) counter for sending packets to a particular recipient. This + * can be used for link quality monitoring and reporting and has no crypto + * impact as it does not increase the likelihood of an IV collision. (The + * crypto we use is not sensitive to the nature of the IV, only that it does + * not repeat.) + * * The flags/cipher/hops bit field is: FFCCCHHH where C is a 3-bit cipher * selection allowing up to 7 cipher suites, F is outside-envelope flags, * and H is hop count. @@ -1102,10 +1110,8 @@ public: }; #ifdef ZT_TRACE - static const char *verbString(Verb v) - throw(); - static const char *errorString(ErrorCode e) - throw(); + static const char *verbString(Verb v); + static const char *errorString(ErrorCode e); #endif template @@ -1303,6 +1309,12 @@ public: /** * Get this packet's unique ID (the IV field interpreted as uint64_t) * + * Note that the least significant 3 bits of this ID will change when armor() + * is called to armor the packet for transport. This is because armor() will + * mask the last 3 bits against the send counter for QoS monitoring use prior + * to actually using the IV to encrypt and MAC the packet. Be aware of this + * when grabbing the packetId of a new packet prior to armor/send. + * * @return Packet ID */ inline uint64_t packetId() const { return at(ZT_PACKET_IDX_IV); } @@ -1337,8 +1349,9 @@ public: * * @param key 32-byte key * @param encryptPayload If true, encrypt packet payload, else just MAC + * @param counter Packet send counter for destination peer -- only least significant 3 bits are used */ - void armor(const void *key,bool encryptPayload); + void armor(const void *key,bool encryptPayload,unsigned int counter); /** * Verify and (if encrypted) decrypt packet diff --git a/node/Path.hpp b/node/Path.hpp index 5993be69..626f2f4f 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -105,6 +105,7 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _outgoingPacketCounter(0), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -115,6 +116,7 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _outgoingPacketCounter(0), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -241,10 +243,18 @@ public: */ inline uint64_t lastIn() const { return _lastIn; } + /** + * Return and increment outgoing packet counter (used with Packet::armor()) + * + * @return Next value that should be used for outgoing packet counter (only least significant 3 bits are used) + */ + inline unsigned int nextOutgoingCounter() { return _outgoingPacketCounter++; } + private: uint64_t _lastOut; uint64_t _lastIn; uint64_t _lastTrustEstablishedPacketReceived; + unsigned int _outgoingPacketCounter; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often diff --git a/node/Peer.cpp b/node/Peer.cpp index 25efab42..c4c8774e 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -101,7 +101,7 @@ void Peer::received( outp.append(redirectTo.rawIpData(),16); } outp.append((uint16_t)redirectTo.port()); - outp.armor(_key,true); + outp.armor(_key,true,path->nextOutgoingCounter()); path->send(RR,outp.data(),outp.size(),now); } else { // For older peers we use RENDEZVOUS to coax them into contacting us elsewhere. @@ -116,7 +116,7 @@ void Peer::received( outp.append((uint8_t)16); outp.append(redirectTo.rawIpData(),16); } - outp.armor(_key,true); + outp.armor(_key,true,path->nextOutgoingCounter()); path->send(RR,outp.data(),outp.size(),now); } suboptimalPath = true; @@ -203,7 +203,7 @@ void Peer::received( #endif } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(path->localAddress(),path->address(),now,true); + attemptToContactAt(path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); path->sent(now); } } @@ -277,7 +277,7 @@ void Peer::received( if (count) { outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); - outp.armor(_key,true); + outp.armor(_key,true,path->nextOutgoingCounter()); path->send(RR,outp.data(),outp.size(),now); } } @@ -342,7 +342,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) } } -void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now) +void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -383,22 +383,22 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u RR->node->expectReplyTo(outp.packetId()); if (atAddress) { - outp.armor(_key,false); // false == don't encrypt full payload, but add MAC + outp.armor(_key,false,counter); // false == don't encrypt full payload, but add MAC RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } else { RR->sw->send(outp,false); // false == don't encrypt full payload, but add MAC } } -void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello) +void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) { if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); RR->node->expectReplyTo(outp.packetId()); - outp.armor(_key,true); + outp.armor(_key,true,counter); RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); } else { - sendHELLO(localAddr,atAddress,now); + sendHELLO(localAddr,atAddress,now,counter); } } @@ -408,7 +408,7 @@ void Peer::tryMemorizedPath(uint64_t now) _lastTriedMemorizedPath = now; InetAddress mp; if (RR->node->externalPathLookup(_id.address(),-1,mp)) - attemptToContactAt(InetAddress(),mp,now,true); + attemptToContactAt(InetAddress(),mp,now,true,0); } } @@ -430,7 +430,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) if (bestp >= 0) { if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { - attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false); + attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); _paths[bestp].path->sent(now); } return true; @@ -454,7 +454,7 @@ void Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uin Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now,false); + attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); _paths[p].path->sent(now); _paths[p].lastReceive = 0; // path will not be used unless it speaks again } diff --git a/node/Peer.hpp b/node/Peer.hpp index a3ec0088..783f48b8 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -150,8 +150,9 @@ public: * @param localAddr Local address * @param atAddress Destination address * @param now Current time + * @param counter Outgoing packet counter */ - void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now); + void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter); /** * Send ECHO (or HELLO for older peers) to this peer at the given address @@ -162,8 +163,9 @@ public: * @param atAddress Destination address * @param now Current time * @param sendFullHello If true, always send a full HELLO instead of just an ECHO + * @param counter Outgoing packet counter */ - void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello); + void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); /** * Try a memorized or statically defined path if any are known diff --git a/node/Switch.cpp b/node/Switch.cpp index 346091a4..bf309e36 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -88,7 +88,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from if ((now - _lastBeaconResponse) >= 2500) { // limit rate of responses _lastBeaconResponse = now; Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP); - outp.armor(peer->key(),true); + outp.armor(peer->key(),true,path->nextOutgoingCounter()); path->send(RR,outp.data(),outp.size(),now); } } @@ -777,7 +777,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) if ((clusterMostRecentMemberId < 0)||(viaPath->lastIn() > clusterMostRecentTs)) { #endif if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now,false); + peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); viaPath->sent(now); } #ifdef ZT_ENABLE_CLUSTER @@ -825,14 +825,14 @@ bool Switch::_trySend(Packet &packet,bool encrypt) if (trustedPathId) { packet.setTrusted(trustedPathId); } else { - packet.armor((clusterMostRecentMemberId >= 0) ? clusterPeerSecret : peer->key(),encrypt); + packet.armor((clusterMostRecentMemberId >= 0) ? clusterPeerSecret : peer->key(),encrypt,(viaPath) ? viaPath->nextOutgoingCounter() : 0); } #else const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address()); if (trustedPathId) { packet.setTrusted(trustedPathId); } else { - packet.armor(peer->key(),encrypt); + packet.armor(peer->key(),encrypt,viaPath->nextOutgoingCounter()); } #endif -- cgit v1.2.3 From 1d39be61b267a85adebeee9e979bd1d84f55da3c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 14:36:52 -0800 Subject: ZeroTier now has link quality measurement. We are not using this yet but decided to put it in to prep for future QoS support and SD-WAN stuff. --- include/ZeroTierOne.h | 10 ++++++++ node/Node.cpp | 3 ++- node/Packet.hpp | 13 ++++++++--- node/Path.hpp | 60 ++++++++++++++++++++++++++++++++++++++++++++---- node/Peer.cpp | 3 +++ node/Switch.cpp | 24 +++++++++---------- node/Utils.hpp | 14 +++++++++++ one.cpp | 3 ++- service/ControlPlane.cpp | 3 ++- 9 files changed, 111 insertions(+), 22 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 2c141f47..5b478afb 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -179,6 +179,11 @@ extern "C" { */ #define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48) +/** + * Maximum value for link quality (min is 0) + */ +#define ZT_PATH_LINK_QUALITY_MAX 0xff + /** * Packet characteristics flag: packet direction, 1 if inbound 0 if outbound */ @@ -1036,6 +1041,11 @@ typedef struct */ uint64_t trustedPathId; + /** + * Path link quality from 0 to 255 (always 255 if peer does not support) + */ + int linkQuality; + /** * Is path expired? */ diff --git a/node/Node.cpp b/node/Node.cpp index 35940d27..a75a56b4 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -401,9 +401,10 @@ ZT_PeerList *Node::peers() const memcpy(&(p->paths[p->pathCount].address),&(path->first->address()),sizeof(struct sockaddr_storage)); p->paths[p->pathCount].lastSend = path->first->lastOut(); p->paths[p->pathCount].lastReceive = path->first->lastIn(); + p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->first->address()); + p->paths[p->pathCount].linkQuality = (int)path->first->linkQuality(); p->paths[p->pathCount].expired = path->second; p->paths[p->pathCount].preferred = (path->first == bestp) ? 1 : 0; - p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->first->address()); ++p->pathCount; } } diff --git a/node/Packet.hpp b/node/Packet.hpp index 2017ce8e..d5817708 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -59,15 +59,17 @@ * + Otherwise backward compatible with protocol v4 * 6 - 1.1.5 ... 1.1.10 * + Network configuration format revisions including binary values - * 7 - 1.1.10 -- 1.2.0 + * 7 - 1.1.10 ... 1.1.17 * + Introduce trusted paths for local SDN use - * 8 - 1.2.0 -- CURRENT + * 8 - 1.1.17 ... 1.2.0 * + Multipart network configurations for large network configs * + Tags and Capabilities * + Inline push of CertificateOfMembership deprecated * + Certificates of representation for federation and mesh + * 9 - 1.2.0 ... CURRENT + * + In-band encoding of packet counter for link quality measurement */ -#define ZT_PROTO_VERSION 8 +#define ZT_PROTO_VERSION 9 /** * Minimum supported protocol version @@ -1319,6 +1321,11 @@ public: */ inline uint64_t packetId() const { return at(ZT_PACKET_IDX_IV); } + /** + * @return Value of link quality counter extracted from this packet's ID, range 0 to 7 (3 bits) + */ + inline unsigned int linkQualityCounter() const { return (unsigned int)(reinterpret_cast(data())[7] & 7); } + /** * Set packet verb * diff --git a/node/Path.hpp b/node/Path.hpp index 626f2f4f..dd6455d1 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -30,6 +31,7 @@ #include "SharedPtr.hpp" #include "AtomicCounter.hpp" #include "NonCopyable.hpp" +#include "Utils.hpp" /** * Maximum return value of preferenceRank() @@ -105,22 +107,34 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _incomingLinkQualityFastLog(0xffffffffffffffffULL), + _incomingLinkQualitySlowLogPtr(0), + _incomingLinkQualitySlowLogCounter(-64), // discard first fast log + _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) { + for(int i=0;i<(int)sizeof(_incomingLinkQualitySlowLog);++i) + _incomingLinkQualitySlowLog[i] = ZT_PATH_LINK_QUALITY_MAX; } Path(const InetAddress &localAddress,const InetAddress &addr) : _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _incomingLinkQualityFastLog(0xffffffffffffffffULL), + _incomingLinkQualitySlowLogPtr(0), + _incomingLinkQualitySlowLogCounter(-64), // discard first fast log + _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) { + for(int i=0;i<(int)sizeof(_incomingLinkQualitySlowLog);++i) + _incomingLinkQualitySlowLog[i] = ZT_PATH_LINK_QUALITY_MAX; } /** @@ -130,6 +144,39 @@ public: */ inline void received(const uint64_t t) { _lastIn = t; } + /** + * Update link quality using a counter from an incoming packet (or packet head in fragmented case) + * + * @param counter Packet link quality counter (range 0 to 7, must not have other bits set) + */ + inline void updateLinkQuality(const unsigned int counter) + { + const unsigned int prev = _incomingLinkQualityPreviousPacketCounter; + _incomingLinkQualityPreviousPacketCounter = counter; + const uint64_t fl = (_incomingLinkQualityFastLog = ((_incomingLinkQualityFastLog << 1) | (uint64_t)(prev == ((counter - 1) & 0x7)))); + if (++_incomingLinkQualitySlowLogCounter >= 64) { + _incomingLinkQualitySlowLogCounter = 0; + _incomingLinkQualitySlowLog[_incomingLinkQualitySlowLogPtr++ % sizeof(_incomingLinkQualitySlowLog)] = Utils::countBits(fl); + } + } + + /** + * @return Link quality from 0 (min) to 255 (max) + */ + inline unsigned int linkQuality() const + { + unsigned long slsize = _incomingLinkQualitySlowLogPtr; + if (slsize > (unsigned long)sizeof(_incomingLinkQualitySlowLog)) + slsize = (unsigned long)sizeof(_incomingLinkQualitySlowLog); + else if (!slsize) + return 255; // ZT_PATH_LINK_QUALITY_MAX + unsigned long lq = 0; + for(unsigned long i=0;i= 255) ? 255 : lq); + } + /** * Set time last trusted packet was received (done in Peer::received()) */ @@ -251,13 +298,18 @@ public: inline unsigned int nextOutgoingCounter() { return _outgoingPacketCounter++; } private: - uint64_t _lastOut; - uint64_t _lastIn; - uint64_t _lastTrustEstablishedPacketReceived; - unsigned int _outgoingPacketCounter; + volatile uint64_t _lastOut; + volatile uint64_t _lastIn; + volatile uint64_t _lastTrustEstablishedPacketReceived; + volatile uint64_t _incomingLinkQualityFastLog; + volatile unsigned long _incomingLinkQualitySlowLogPtr; + volatile signed int _incomingLinkQualitySlowLogCounter; + volatile unsigned int _incomingLinkQualityPreviousPacketCounter; + volatile unsigned int _outgoingPacketCounter; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often + volatile uint8_t _incomingLinkQualitySlowLog[32]; AtomicCounter __refCount; }; diff --git a/node/Peer.cpp b/node/Peer.cpp index c4c8774e..1dde8b65 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -142,6 +142,9 @@ void Peer::received( } if (hops == 0) { + if (_vProto >= 9) + path->updateLinkQuality((unsigned int)(packetId & 7)); + bool pathIsConfirmed = false; { Mutex::Lock _l(_paths_m); diff --git a/node/Switch.cpp b/node/Switch.cpp index bf309e36..0392aec1 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -185,17 +185,6 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } else if (len >= ZT_PROTO_MIN_PACKET_LENGTH) { // min length check is important! // Handle packet head ------------------------------------------------- - // See packet format in Packet.hpp to understand this - const uint64_t packetId = ( - (((uint64_t)reinterpret_cast(data)[0]) << 56) | - (((uint64_t)reinterpret_cast(data)[1]) << 48) | - (((uint64_t)reinterpret_cast(data)[2]) << 40) | - (((uint64_t)reinterpret_cast(data)[3]) << 32) | - (((uint64_t)reinterpret_cast(data)[4]) << 24) | - (((uint64_t)reinterpret_cast(data)[5]) << 16) | - (((uint64_t)reinterpret_cast(data)[6]) << 8) | - ((uint64_t)reinterpret_cast(data)[7]) - ); const Address destination(reinterpret_cast(data) + 8,ZT_ADDRESS_LENGTH); const Address source(reinterpret_cast(data) + 13,ZT_ADDRESS_LENGTH); @@ -297,6 +286,17 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } else if ((reinterpret_cast(data)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED) != 0) { // Packet is the head of a fragmented packet series + const uint64_t packetId = ( + (((uint64_t)reinterpret_cast(data)[0]) << 56) | + (((uint64_t)reinterpret_cast(data)[1]) << 48) | + (((uint64_t)reinterpret_cast(data)[2]) << 40) | + (((uint64_t)reinterpret_cast(data)[3]) << 32) | + (((uint64_t)reinterpret_cast(data)[4]) << 24) | + (((uint64_t)reinterpret_cast(data)[5]) << 16) | + (((uint64_t)reinterpret_cast(data)[6]) << 8) | + ((uint64_t)reinterpret_cast(data)[7]) + ); + Mutex::Lock _l(_rxQueue_m); RXQueueEntry *const rq = _findRXQueueEntry(now,packetId); @@ -344,7 +344,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from rq = tmp; } rq->timestamp = now; - rq->packetId = packetId; + rq->packetId = packet.packetId(); rq->frag0 = packet; rq->totalFragments = 1; rq->haveFragments = 1; diff --git a/node/Utils.hpp b/node/Utils.hpp index 7b1994be..ceb29d7e 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -252,6 +252,20 @@ public: return ((((v + (v >> 4)) & (uint32_t)0xF0F0F0F) * (uint32_t)0x1010101) >> 24); } + /** + * Count the number of bits set in an integer + * + * @param v 64-bit integer + * @return Number of bits set in this integer (0-64) + */ + static inline uint64_t countBits(uint64_t v) + { + v = v - ((v >> 1) & (uint64_t)~(uint64_t)0/3); + v = (v & (uint64_t)~(uint64_t)0/15*3) + ((v >> 2) & (uint64_t)~(uint64_t)0/15*3); + v = (v + (v >> 4)) & (uint64_t)~(uint64_t)0/255*15; + return (uint64_t)(v * ((uint64_t)~(uint64_t)0/255)) >> 56; + } + /** * Check if a memory buffer is all-zero * diff --git a/one.cpp b/one.cpp index 3649c18c..8f116aa0 100644 --- a/one.cpp +++ b/one.cpp @@ -355,7 +355,8 @@ static int cli(int argc,char **argv) char tmp[256]; std::string addr = path["address"]; const uint64_t now = OSUtils::now(); - Utils::snprintf(tmp,sizeof(tmp),"%s;%llu;%llu",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"]); + const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; + Utils::snprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); bestPath = tmp; break; } diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 9ba001ea..e995a4a5 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -127,10 +127,11 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(); j["lastSend"] = peer->paths[i].lastSend; j["lastReceive"] = peer->paths[i].lastReceive; + j["trustedPathId"] = peer->paths[i].trustedPathId; + j["linkQuality"] = (double)peer->paths[i].linkQuality / (double)ZT_PATH_LINK_QUALITY_MAX; j["active"] = (bool)(peer->paths[i].expired == 0); j["expired"] = (bool)(peer->paths[i].expired != 0); j["preferred"] = (bool)(peer->paths[i].preferred != 0); - j["trustedPathId"] = peer->paths[i].trustedPathId; pa.push_back(j); } pj["paths"] = pa; -- cgit v1.2.3 From 592b628523029b0a821952998dc10f1c7462dd3e Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Wed, 1 Mar 2017 14:50:28 -0800 Subject: comment broken TRACE message --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index dc976f03..5b8c7d59 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -647,7 +647,7 @@ static _doZtFilterResult _doZtFilter( const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + // FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); -- cgit v1.2.3 From d79585d44d54256ee3ddbec264174f9b6e008bed Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 15:12:17 -0800 Subject: Circuit tests now report link quality. Also fixed a little thing in revocation propagation. --- include/ZeroTierOne.h | 5 +++++ node/IncomingPacket.cpp | 18 +++++++++++------- node/Network.cpp | 1 + node/Packet.hpp | 3 +-- 4 files changed, 18 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 5b478afb..98413a21 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1270,6 +1270,11 @@ typedef struct { */ struct sockaddr_storage receivedFromRemoteAddress; + /** + * Path link quality of physical path over which test was received + */ + int receivedFromLinkQuality; + /** * Next hops to which packets are being or will be sent by the reporter * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 85b06d50..9c13a283 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1342,7 +1342,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt outp.append((uint8_t)hops()); _path->localAddress().serialize(outp); _path->address().serialize(outp); - outp.append((uint16_t)0); // no additional fields + outp.append((uint16_t)_path->linkQuality()); outp.append((uint8_t)breadth); for(unsigned int h=0;h(&(report.receivedOnLocalAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58); const unsigned int receivedFromRemoteAddressLen = reinterpret_cast(&(report.receivedFromRemoteAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen); + unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen + receivedFromRemoteAddressLen; + if (report.protocolVersion >= 9) { + report.receivedFromLinkQuality = at(ptr); ptr += 2; + } else { + report.receivedFromLinkQuality = ZT_PATH_LINK_QUALITY_MAX; + ptr += at(ptr) + 2; // this field was once an 'extended field length' reserved field, which was always set to 0 + } - unsigned int nhptr = ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen + receivedFromRemoteAddressLen; - nhptr += at(nhptr) + 2; // add "additional field" length, which right now will be zero - - report.nextHopCount = (*this)[nhptr++]; + report.nextHopCount = (*this)[ptr++]; if (report.nextHopCount > ZT_CIRCUIT_TEST_MAX_HOP_BREADTH) // sanity check, shouldn't be possible report.nextHopCount = ZT_CIRCUIT_TEST_MAX_HOP_BREADTH; for(unsigned int h=0;h(&(report.nextHops[h].physicalAddress))->deserialize(*this,nhptr); + report.nextHops[h].address = Address(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); ptr += ZT_ADDRESS_LENGTH; + ptr += reinterpret_cast(&(report.nextHops[h].physicalAddress))->deserialize(*this,ptr); } RR->node->postCircuitTestReport(&report); diff --git a/node/Network.cpp b/node/Network.cpp index dc976f03..e4b91bb6 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1422,6 +1422,7 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c outp.append((uint16_t)0); // no capabilities outp.append((uint16_t)0); // no tags outp.append((uint16_t)1); // one revocation! + outp.append((uint16_t)0); // no certificates of ownership rev.serialize(outp); RR->sw->send(outp,true); } diff --git a/node/Packet.hpp b/node/Packet.hpp index d5817708..87863b19 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1041,8 +1041,7 @@ public: * <[1] 8-bit packet hop count of received CIRCUIT_TEST> * <[...] local wire address on which packet was received> * <[...] remote wire address from which packet was received> - * <[2] 16-bit length of additional fields> - * <[...] additional fields> + * <[2] 16-bit path link quality of path over which packet was received> * <[1] 8-bit number of next hops (breadth)> * <[...] next hop information> * -- cgit v1.2.3 From 136fddc7f1d965aa5f4e3699195b4333131747a3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 15:14:57 -0800 Subject: Fix FILTER_TRACE breakage. --- node/Network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 51528150..9223987c 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -647,7 +647,7 @@ static _doZtFilterResult _doZtFilter( const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value); - // FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); + FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,localTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); -- cgit v1.2.3 From a577b8d3816069a448a946302048a377b55cd74a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 1 Mar 2017 16:33:34 -0800 Subject: Update how controller handles circuit tests -- save results to filesystem. --- controller/EmbeddedNetworkController.cpp | 137 +++++++++++++++---------------- controller/EmbeddedNetworkController.hpp | 15 ++-- controller/JSONDB.cpp | 25 ++++-- controller/JSONDB.hpp | 6 +- controller/README.md | 11 +-- node/Peer.cpp | 6 +- service/OneService.cpp | 2 +- 7 files changed, 100 insertions(+), 102 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 0e57fc14..7915765b 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -428,9 +428,9 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return false; } -EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath,FILE *feed) : +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _threadsStarted(false), - _db(dbPath,feed), + _db(dbPath), _node(node) { OSUtils::mkdir(dbPath); @@ -546,21 +546,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( return 200; } - } else if ((path[2] == "test")&&(path.size() >= 4)) { - - Mutex::Lock _l(_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); - if ((cte != _circuitTests.end())&&(cte->second.test)) { - - responseBody = "["; - responseBody.append(cte->second.jsonResults); - responseBody.push_back(']'); - responseContentType = "application/json"; - - return 200; - - } // else 404 - } // else 404 } else { @@ -755,9 +740,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( return 200; } else if ((path.size() == 3)&&(path[2] == "test")) { - Mutex::Lock _l(_circuitTests_m); + Mutex::Lock _l(_tests_m); - ZT_CircuitTest *test = (ZT_CircuitTest *)malloc(sizeof(ZT_CircuitTest)); + _tests.push_back(ZT_CircuitTest()); + ZT_CircuitTest *const test = &(_tests.back()); memset(test,0,sizeof(ZT_CircuitTest)); Utils::getSecureRandom(&(test->testId),sizeof(test->testId)); @@ -781,7 +767,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( test->reportAtEveryHop = (OSUtils::jsonBool(b["reportAtEveryHop"],true) ? 1 : 0); if (!test->hopCount) { - ::free((void *)test); + _tests.pop_back(); responseBody = "{ \"message\": \"a test must contain at least one hop\" }"; responseContentType = "application/json"; return 400; @@ -789,18 +775,18 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( test->timestamp = OSUtils::now(); - _CircuitTestEntry &te = _circuitTests[test->testId]; - te.test = test; - te.jsonResults = ""; - - if (_node) + if (_node) { _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); - else return 500; + } else { + _tests.pop_back(); + return 500; + } char json[1024]; Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); responseBody = json; responseContentType = "application/json"; + return 200; } // else 404 @@ -1137,62 +1123,67 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::threadMain() throw() { + uint64_t lastCircuitTestCheck = 0; for(;;) { - _RQEntry *const qe = _queue.get(); + _RQEntry *const qe = _queue.get(); // waits on next request if (!qe) break; // enqueue a NULL to terminate threads try { _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); } catch ( ... ) {} delete qe; + + uint64_t now = OSUtils::now(); + if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + lastCircuitTestCheck = now; + Mutex::Lock _l(_tests_m); + for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { + if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + _node->circuitTestEnd(&(*i)); + _tests.erase(i++); + } else ++i; + } + } } } void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) { - char tmp[65535]; + char tmp[1024],id[128]; EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); - if (!test) - return; - if (!report) - return; - - Mutex::Lock _l(self->_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); - - if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? - self->_node->circuitTestEnd(test); - ::free((void *)test); - return; - } + if ((!test)||(!report)||(!test->credentialNetworkId)) return; // sanity check + const uint64_t now = OSUtils::now(); + Utils::snprintf(id,sizeof(id),"network/%.16llx/test/%.16llx-%.16llx-%.10llx-%.10llx",test->credentialNetworkId,test->testId,now,report->upstream,report->current); Utils::snprintf(tmp,sizeof(tmp), - "%s{\n" - "\t\"timestamp\": %llu," ZT_EOL_S - "\t\"testId\": \"%.16llx\"," ZT_EOL_S - "\t\"upstream\": \"%.10llx\"," ZT_EOL_S - "\t\"current\": \"%.10llx\"," ZT_EOL_S - "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S - "\t\"flags\": %llu," ZT_EOL_S - "\t\"sourcePacketHopCount\": %u," ZT_EOL_S - "\t\"errorCode\": %u," ZT_EOL_S - "\t\"vendor\": %d," ZT_EOL_S - "\t\"protocolVersion\": %u," ZT_EOL_S - "\t\"majorVersion\": %u," ZT_EOL_S - "\t\"minorVersion\": %u," ZT_EOL_S - "\t\"revision\": %u," ZT_EOL_S - "\t\"platform\": %d," ZT_EOL_S - "\t\"architecture\": %d," ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S - "}", - ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), - (unsigned long long)report->timestamp, + "{\"id\": \"%s\"," + "\"timestamp\": %llu," + "\"networkId\": \"%.16llx\"," + "\"testId\": \"%.16llx\"," + "\"upstream\": \"%.10llx\"," + "\"current\": \"%.10llx\"," + "\"receivedTimestamp\": %llu," + "\"sourcePacketId\": \"%.16llx\"," + "\"flags\": %llu," + "\"sourcePacketHopCount\": %u," + "\"errorCode\": %u," + "\"vendor\": %d," + "\"protocolVersion\": %u," + "\"majorVersion\": %u," + "\"minorVersion\": %u," + "\"revision\": %u," + "\"platform\": %d," + "\"architecture\": %d," + "\"receivedOnLocalAddress\": \"%s\"," + "\"receivedFromRemoteAddress\": \"%s\"," + "\"receivedFromLinkQuality\": %f}", + id + 30, // last bit only, not leading path + (unsigned long long)test->timestamp, + (unsigned long long)test->credentialNetworkId, (unsigned long long)test->testId, (unsigned long long)report->upstream, (unsigned long long)report->current, - (unsigned long long)OSUtils::now(), + (unsigned long long)now, (unsigned long long)report->sourcePacketId, (unsigned long long)report->flags, report->sourcePacketHopCount, @@ -1205,9 +1196,11 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes (int)report->platform, (int)report->architecture, reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); + reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str(), + ((double)report->receivedFromLinkQuality / (double)ZT_PATH_LINK_QUALITY_MAX)); - cte->second.jsonResults.append(tmp); + Mutex::Lock _l(self->_db_m); + self->_db.writeRaw(id,std::string(tmp)); } void EmbeddedNetworkController::_request( @@ -1354,12 +1347,12 @@ void EmbeddedNetworkController::_request( if (requestPacketId) { // only log if this is a request, not for generated pushes json rlEntry = json::object(); rlEntry["ts"] = now; - rlEntry["authorized"] = (authorizedBy) ? true : false; - rlEntry["authorizedBy"] = (authorizedBy) ? authorizedBy : ""; - rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); + rlEntry["auth"] = (authorizedBy) ? true : false; + rlEntry["authBy"] = (authorizedBy) ? authorizedBy : ""; + rlEntry["vMajor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + rlEntry["vMinor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + rlEntry["vRev"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + rlEntry["vProto"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); if (fromAddr) rlEntry["fromAddr"] = fromAddr.toString(); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 3e39eaf5..ab7cdd53 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -46,6 +46,9 @@ // Number of background threads to start -- not actually started until needed #define ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT 2 +// TTL for circuit tests +#define ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION 120000 + namespace ZeroTier { class Node; @@ -56,9 +59,8 @@ public: /** * @param node Parent node * @param dbPath Path to store data - * @param feed FILE to send feed of all data and changes to (zero-delimited JSON objects) or NULL for none */ - EmbeddedNetworkController(Node *node,const char *dbPath,FILE *feed); + EmbeddedNetworkController(Node *node,const char *dbPath); virtual ~EmbeddedNetworkController(); virtual void init(const Identity &signingId,Sender *sender); @@ -199,13 +201,8 @@ private: NetworkController::Sender *_sender; Identity _signingId; - struct _CircuitTestEntry - { - ZT_CircuitTest *test; - std::string jsonResults; - }; - std::map< uint64_t,_CircuitTestEntry > _circuitTests; - Mutex _circuitTests_m; + std::list< ZT_CircuitTest > _tests; + Mutex _tests_m; std::map< std::pair,uint64_t > _lastRequestTime; Mutex _lastRequestTime_m; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 044f791c..1277aabb 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -22,6 +22,22 @@ namespace ZeroTier { static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); +bool JSONDB::writeRaw(const std::string &n,const std::string &obj) +{ + if (!_isValidObjectName(n)) + return false; + + const std::string path(_genPath(n,true)); + if (!path.length()) + return false; + + const std::string buf(obj); + if (!OSUtils::writeFile(path.c_str(),buf)) + return false; + + return true; +} + bool JSONDB::put(const std::string &n,const nlohmann::json &obj) { if (!_isValidObjectName(n)) @@ -35,9 +51,6 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) if (!OSUtils::writeFile(path.c_str(),buf)) return false; - if (_feed) - fwrite(buf.c_str(),buf.length()+1,1,_feed); - _E &e = _db[n]; e.obj = obj; e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); @@ -72,9 +85,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe e->second.obj = OSUtils::jsonParse(buf); e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP e->second.lastCheck = now; - - if (_feed) - fwrite(buf.c_str(),buf.length()+1,1,_feed); // it changed, so send to feed (also sends all objects on startup, which we want for Central) } catch ( ... ) {} // parse errors result in "holding pattern" behavior } } @@ -99,9 +109,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe e2.lastModifiedOnDisk = lm; e2.lastCheck = now; - if (_feed) - fwrite(buf.c_str(),buf.length()+1,1,_feed); - return e2.obj; } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 40113655..5b7c5e50 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -42,8 +42,7 @@ namespace ZeroTier { class JSONDB { public: - JSONDB(const std::string &basePath,FILE *feed) : - _feed(feed), + JSONDB(const std::string &basePath) : _basePath(basePath) { _reload(_basePath); @@ -55,6 +54,8 @@ public: _reload(_basePath); } + bool writeRaw(const std::string &n,const std::string &obj); + bool put(const std::string &n,const nlohmann::json &obj); inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } @@ -108,7 +109,6 @@ private: inline bool operator!=(const _E &e) const { return (obj != e.obj); } }; - FILE *_feed; std::string _basePath; std::map _db; }; diff --git a/controller/README.md b/controller/README.md index 093300a6..db8d0153 100644 --- a/controller/README.md +++ b/controller/README.md @@ -237,11 +237,12 @@ Note that managed IP assignments are only used if they fall within a managed rou | Field | Type | Description | | --------------------- | ------------- | ------------------------------------------------- | | ts | integer | Time of request, ms since epoch | -| authorized | boolean | Was member authorized? | -| clientMajorVersion | integer | Client major version or -1 if unknown | -| clientMinorVersion | integer | Client minor version or -1 if unknown | -| clientRevision | integer | Client revision or -1 if unknown | -| clientProtocolVersion | integer | ZeroTier protocol version reported by client | +| auth | boolean | Was member authorized? | +| authBy | string | How was member authorized? | +| vMajor | integer | Client major version or -1 if unknown | +| vMinor | integer | Client minor version or -1 if unknown | +| vRev | integer | Client revision or -1 if unknown | +| vProto | integer | ZeroTier protocol version reported by client | | fromAddr | string | Physical address if known | The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/node/Peer.cpp b/node/Peer.cpp index 1dde8b65..fa3ce6c8 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -141,10 +141,10 @@ void Peer::received( path->trustedPacketReceived(now); } - if (hops == 0) { - if (_vProto >= 9) - path->updateLinkQuality((unsigned int)(packetId & 7)); + if (_vProto >= 9) + path->updateLinkQuality((unsigned int)(packetId & 7)); + if (hops == 0) { bool pathIsConfirmed = false; { Mutex::Lock _l(_paths_m); diff --git a/service/OneService.cpp b/service/OneService.cpp index 81950e26..8d8856a2 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -638,7 +638,7 @@ public: return _termReason; } - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str()); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER -- cgit v1.2.3 From d56f740dc6cf35bd4e26c17503170d0f6c8035ec Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 3 Mar 2017 13:49:21 -0800 Subject: Now with less bugs. --- node/IncomingPacket.cpp | 6 ++++++ node/Node.hpp | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 9c13a283..856538b4 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -875,6 +875,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } + if (p >= size()) return true; + const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i= size()) return true; + const unsigned int numRevocations = at(p); p += 2; for(unsigned int i=0;i= size()) return true; + const unsigned int numCoos = at(p); p += 2; for(unsigned int i=0;i> 32); + const unsigned long pid2 = (unsigned long)(packetId >> 32); + const unsigned long bucket = (unsigned long)(pid2 & ZT_EXPECTING_REPLIES_BUCKET_MASK1); + _expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = (uint32_t)pid2; } /** @@ -243,10 +244,10 @@ public: */ inline bool expectingReplyTo(const uint64_t packetId) const { - const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1); - const uint32_t pid = (uint32_t)(packetId >> 32); + const uint32_t pid2 = (uint32_t)(packetId >> 32); + const unsigned long bucket = (unsigned long)(pid2 & ZT_EXPECTING_REPLIES_BUCKET_MASK1); for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) { - if (_expectingRepliesTo[bucket][i] == pid) + if (_expectingRepliesTo[bucket][i] == pid2) return true; } return false; -- cgit v1.2.3 From 5e6a4e5f5e0022dccbc2f6cf8a8b38c038720866 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 6 Mar 2017 15:12:28 -0800 Subject: Send revocations automatically on deauth for instant kill, also fix some issues with the RP. --- controller/EmbeddedNetworkController.cpp | 16 ++++++++++++++-- node/Membership.hpp | 2 +- node/Network.cpp | 2 +- node/NetworkController.hpp | 11 ++++++++++- node/Node.cpp | 18 ++++++++++++++++++ node/Node.hpp | 1 + node/Packet.hpp | 3 +-- node/Revocation.hpp | 2 +- 8 files changed, 47 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 78fa79f2..2f6142a9 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -661,6 +661,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( ah["ct"] = json(); ah["c"] = json(); member["authHistory"].push_back(ah); + + // Member is being de-authorized, so spray Revocation objects to all online members + if (!newAuth) { + Revocation rev(_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); + rev.sign(_signingId); + Mutex::Lock _l(_lastRequestTime_m); + for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { + if ((now - i->second) < ZT_NETWORK_AUTOCONF_DELAY) + _node->ncSendRevocation(Address(i->first.first),rev); + } + } } } @@ -1037,8 +1048,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Mutex::Lock _l(_db_m); _db.put("network",nwids,network); } - std::string pfx("network/"); pfx.append(nwids); pfx.append("/member/"); - _db.filter(pfx,120000,[this,&now,&nwid](const std::string &n,const json &obj) { + + // Send an update to all members of the network + _db.filter((std::string("network/") + nwids + "/member/"),120000,[this,&now,&nwid](const std::string &n,const json &obj) { _pushMemberUpdate(now,nwid,obj); return true; // do not delete }); diff --git a/node/Membership.hpp b/node/Membership.hpp index a7794328..97510b57 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -191,7 +191,7 @@ public: { if (nconf.isPublic()) return true; - if ((_comRevocationThreshold)&&(_com.timestamp().first <= _comRevocationThreshold)) + if (_com.timestamp().first <= _comRevocationThreshold) return false; return nconf.com.agreesWith(_com); } diff --git a/node/Network.cpp b/node/Network.cpp index 9223987c..dd812cab 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1422,8 +1422,8 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c outp.append((uint16_t)0); // no capabilities outp.append((uint16_t)0); // no tags outp.append((uint16_t)1); // one revocation! - outp.append((uint16_t)0); // no certificates of ownership rev.serialize(outp); + outp.append((uint16_t)0); // no certificates of ownership RR->sw->send(outp,true); } } diff --git a/node/NetworkController.hpp b/node/NetworkController.hpp index fc5db4af..0634f435 100644 --- a/node/NetworkController.hpp +++ b/node/NetworkController.hpp @@ -24,11 +24,12 @@ #include "Constants.hpp" #include "Dictionary.hpp" #include "NetworkConfig.hpp" +#include "Revocation.hpp" +#include "Address.hpp" namespace ZeroTier { class Identity; -class Address; struct InetAddress; /** @@ -62,6 +63,14 @@ public: */ virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig) = 0; + /** + * Send revocation to a node + * + * @param destination Destination node address + * @param rev Revocation to send + */ + virtual void ncSendRevocation(const Address &destination,const Revocation &rev) = 0; + /** * Send a network configuration request error * diff --git a/node/Node.cpp b/node/Node.cpp index a75a56b4..1125ca7a 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -774,6 +774,24 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de } } +void Node::ncSendRevocation(const Address &destination,const Revocation &rev) +{ + if (destination == RR->identity.address()) { + SharedPtr n(network(rev.networkId())); + if (!n) return; + n->addCredential(RR->identity.address(),rev); + } else { + Packet outp(destination,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); + outp.append((uint8_t)0x00); + outp.append((uint16_t)0); + outp.append((uint16_t)0); + outp.append((uint16_t)1); + rev.serialize(outp); + outp.append((uint16_t)0); + RR->sw->send(outp,true); + } +} + void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) { if (destination == RR->identity.address()) { diff --git a/node/Node.hpp b/node/Node.hpp index ab201f06..21eac617 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -271,6 +271,7 @@ public: } virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig); + virtual void ncSendRevocation(const Address &destination,const Revocation &rev); virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); private: diff --git a/node/Packet.hpp b/node/Packet.hpp index 87863b19..fb332b7d 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -731,8 +731,7 @@ public: /** * Network credentials push: - * <[...] serialized certificate of membership> - * [<[...] additional certificates of membership>] + * [<[...] one or more certificates of membership>] * <[1] 0x00, null byte marking end of COM array> * <[2] 16-bit number of capabilities> * <[...] one or more serialized Capability> diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 3903f440..1697b52f 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -89,8 +89,8 @@ public: { if (signer.hasPrivate()) { Buffer tmp; - this->serialize(tmp,true); _signedBy = signer.address(); + this->serialize(tmp,true); _signature = signer.sign(tmp.data(),tmp.size()); return true; } -- cgit v1.2.3 From a97918f81253c0dbb08766271c6ab6111120bf0d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 7 Mar 2017 13:57:31 -0800 Subject: Windows build fixes. --- ext/installfiles/windows/ZeroTier One.aip | 43 ++++++++++++++++++++++--- node/Path.hpp | 2 +- windows/ZeroTierOne/ZeroTierOne.vcxproj | 2 -- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 6 ---- 4 files changed, 39 insertions(+), 14 deletions(-) (limited to 'node') diff --git a/ext/installfiles/windows/ZeroTier One.aip b/ext/installfiles/windows/ZeroTier One.aip index 3fb6d5c6..18cc0ab7 100644 --- a/ext/installfiles/windows/ZeroTier One.aip +++ b/ext/installfiles/windows/ZeroTier One.aip @@ -47,6 +47,7 @@ + @@ -69,20 +70,25 @@ + - + - + + + + + @@ -92,6 +98,12 @@ + + + + + + @@ -133,6 +145,10 @@ + + + + @@ -161,6 +177,7 @@ + @@ -200,12 +217,19 @@ + + + + + + + @@ -267,13 +291,19 @@ + + + + + + - + @@ -281,7 +311,7 @@ - + @@ -300,12 +330,15 @@ - + + + + diff --git a/node/Path.hpp b/node/Path.hpp index dd6455d1..62f29c22 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -156,7 +156,7 @@ public: const uint64_t fl = (_incomingLinkQualityFastLog = ((_incomingLinkQualityFastLog << 1) | (uint64_t)(prev == ((counter - 1) & 0x7)))); if (++_incomingLinkQualitySlowLogCounter >= 64) { _incomingLinkQualitySlowLogCounter = 0; - _incomingLinkQualitySlowLog[_incomingLinkQualitySlowLogPtr++ % sizeof(_incomingLinkQualitySlowLog)] = Utils::countBits(fl); + _incomingLinkQualitySlowLog[_incomingLinkQualitySlowLogPtr++ % sizeof(_incomingLinkQualitySlowLog)] = (uint8_t)Utils::countBits(fl); } } diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index d0b027fa..5092a655 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -72,7 +72,6 @@ - @@ -151,7 +150,6 @@ - diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index 1b7b469d..ca1640e9 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -78,9 +78,6 @@ - - Source Files\service - Source Files\service @@ -275,9 +272,6 @@ Header Files\osdep - - Header Files\service - Header Files\service -- cgit v1.2.3 From 0c00b8370207c51fbc9c9901cfa0daccc9707295 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 17:34:41 -0800 Subject: cryptField() used to obscure extended fields in HELLO cannot use mangleKey() --- node/IncomingPacket.cpp | 2 +- node/Packet.cpp | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 856538b4..f3ec7505 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -320,7 +320,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut // Get moon IDs and timestamps if present if ((ptr + 2) <= size()) { - unsigned int numMoons = at(ptr); ptr += 2; + const unsigned int numMoons = at(ptr); ptr += 2; for(unsigned int i=0;i(at(ptr),at(ptr + 8))); diff --git a/node/Packet.cpp b/node/Packet.cpp index 82a5d7ea..80ea2de7 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -2025,14 +2025,9 @@ bool Packet::dearmor(const void *key) void Packet::cryptField(const void *key,unsigned int start,unsigned int len) { - unsigned char mangledKey[32]; - unsigned char macKey[32]; - _salsa20MangleKey((const unsigned char *)key,mangledKey); - mangledKey[0] ^= 0x7f; - mangledKey[1] ^= ((start >> 8) & 0xff); - mangledKey[2] ^= (start & 0xff); // slightly alter key for this use case as an added guard against key stream reuse - Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)); - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); // discard the first 32 bytes of key stream (the ones use for MAC in armor()) as a precaution + unsigned char discard[32]; + Salsa20 s20(key,256,field(ZT_PACKET_IDX_IV,8)); + s20.crypt12(ZERO_KEY,discard,sizeof(discard)); // discard the first 32 bytes of key stream (the ones use for MAC in armor()) as a precaution unsigned char *const ptr = field(start,len); s20.crypt12(ptr,ptr,len); } -- cgit v1.2.3 From aad6f79efa9a335f13274232519b65d90f3cd672 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 17:44:25 -0800 Subject: Also must mask off counter bits in IV in cryptField. --- node/Packet.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index 80ea2de7..aaceb9aa 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -2025,9 +2025,11 @@ bool Packet::dearmor(const void *key) void Packet::cryptField(const void *key,unsigned int start,unsigned int len) { - unsigned char discard[32]; - Salsa20 s20(key,256,field(ZT_PACKET_IDX_IV,8)); - s20.crypt12(ZERO_KEY,discard,sizeof(discard)); // discard the first 32 bytes of key stream (the ones use for MAC in armor()) as a precaution + const uint8_t *const data = reinterpret_cast(data()); + uint8_t iv[8]; + for(int i=0;i<8;++i) iv[i] = data[i]; + iv[7] &= 0xf8; // mask off least significant 3 bits of packet ID / IV since this is unset when this function gets called + Salsa20 s20(key,256,data); unsigned char *const ptr = field(start,len); s20.crypt12(ptr,ptr,len); } -- cgit v1.2.3 From ecacdf27a9ad2c15d300a8d61fd0c560030e81dd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 17:45:05 -0800 Subject: Build fix (typo) --- node/Packet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index aaceb9aa..fa1b9cf4 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -2025,7 +2025,7 @@ bool Packet::dearmor(const void *key) void Packet::cryptField(const void *key,unsigned int start,unsigned int len) { - const uint8_t *const data = reinterpret_cast(data()); + const uint8_t *const data = reinterpret_cast(unsafeData()); uint8_t iv[8]; for(int i=0;i<8;++i) iv[i] = data[i]; iv[7] &= 0xf8; // mask off least significant 3 bits of packet ID / IV since this is unset when this function gets called -- cgit v1.2.3 From 47166c9614687f4b67d501328663639766dd56f3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 17:54:14 -0800 Subject: Sigh. Another thinko. --- node/Packet.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index fa1b9cf4..c825ea9b 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -2025,13 +2025,12 @@ bool Packet::dearmor(const void *key) void Packet::cryptField(const void *key,unsigned int start,unsigned int len) { - const uint8_t *const data = reinterpret_cast(unsafeData()); + uint8_t *const data = reinterpret_cast(unsafeData()); uint8_t iv[8]; for(int i=0;i<8;++i) iv[i] = data[i]; iv[7] &= 0xf8; // mask off least significant 3 bits of packet ID / IV since this is unset when this function gets called - Salsa20 s20(key,256,data); - unsigned char *const ptr = field(start,len); - s20.crypt12(ptr,ptr,len); + Salsa20 s20(key,256,iv); + s20.crypt12(data + start,data + start,len); } bool Packet::compress() -- cgit v1.2.3 From db87d95c1d2da059a402bd66c3040ecde8965bb2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 19:31:51 -0800 Subject: getUpstreamPeer issue with interim federated roots --- node/Topology.cpp | 73 ++++++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 47 deletions(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index 5abc4df0..21547cd2 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -159,61 +159,40 @@ void Topology::saveIdentity(const Identity &id) SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid) { const uint64_t now = RR->node->now(); + unsigned int bestQualityOverall = ~((unsigned int)0); + unsigned int bestQualityNotAvoid = ~((unsigned int)0); + const SharedPtr *bestOverall = (const SharedPtr *)0; + const SharedPtr *bestNotAvoid = (const SharedPtr *)0; + Mutex::Lock _l1(_peers_m); Mutex::Lock _l2(_upstreams_m); - if (_amRoot) { - /* If I am a root, pick another root that isn't mine and that - * has a numerically greater ID. This causes packets to roam - * around the top rather than bouncing between just two. */ - - for(unsigned long p=0;p<_upstreamAddresses.size();++p) { - if (_upstreamAddresses[p] == RR->identity.address()) { - for(unsigned long q=1;q<_upstreamAddresses.size();++q) { - const SharedPtr *const nextsn = _peers.get(_upstreamAddresses[(p + q) % _upstreamAddresses.size()]); - if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) - return *nextsn; + for(std::vector

::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { + const SharedPtr *p = _peers.get(*a); + if (p) { + bool avoiding = false; + for(unsigned int i=0;iaddress()) { + avoiding = true; + break; } - break; } - } - - } else { - /* Otherwise pick the bestest looking upstream */ - - unsigned int bestQualityOverall = ~((unsigned int)0); - unsigned int bestQualityNotAvoid = ~((unsigned int)0); - const SharedPtr *bestOverall = (const SharedPtr *)0; - const SharedPtr *bestNotAvoid = (const SharedPtr *)0; - - for(std::vector
::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { - const SharedPtr *p = _peers.get(*a); - if (p) { - bool avoiding = false; - for(unsigned int i=0;iaddress()) { - avoiding = true; - break; - } - } - const unsigned int q = (*p)->relayQuality(now); - if (q <= bestQualityOverall) { - bestQualityOverall = q; - bestOverall = &(*p); - } - if ((!avoiding)&&(q <= bestQualityNotAvoid)) { - bestQualityNotAvoid = q; - bestNotAvoid = &(*p); - } + const unsigned int q = (*p)->relayQuality(now); + if (q <= bestQualityOverall) { + bestQualityOverall = q; + bestOverall = &(*p); + } + if ((!avoiding)&&(q <= bestQualityNotAvoid)) { + bestQualityNotAvoid = q; + bestNotAvoid = &(*p); } } + } - if (bestNotAvoid) { - return *bestNotAvoid; - } else if ((!strictAvoid)&&(bestOverall)) { - return *bestOverall; - } - + if (bestNotAvoid) { + return *bestNotAvoid; + } else if ((!strictAvoid)&&(bestOverall)) { + return *bestOverall; } return SharedPtr(); -- cgit v1.2.3 From e3b1fc2ac060408b1c0ec61aa3215bc03ef18848 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 19:52:08 -0800 Subject: Tweak WHOIS path for federation. --- node/Constants.hpp | 7 ++++++- node/Peer.hpp | 2 +- node/Switch.cpp | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 3bda3805..c2961f12 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -202,7 +202,7 @@ /** * Maximum identity WHOIS retries (each attempt tries consulting a different peer) */ -#define ZT_MAX_WHOIS_RETRIES 3 +#define ZT_MAX_WHOIS_RETRIES 4 /** * Transmit queue entry timeout @@ -390,6 +390,11 @@ */ #define ZT_PEER_CREDEITIALS_CUTOFF_LIMIT 15 +/** + * WHOIS rate limit (we allow these to be pretty fast) + */ +#define ZT_PEER_WHOIS_RATE_LIMIT 50 + /** * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound */ diff --git a/node/Peer.hpp b/node/Peer.hpp index 783f48b8..72040b1d 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -370,7 +370,7 @@ public: */ inline bool rateGateInboundWhoisRequest(const uint64_t now) { - if ((now - _lastWhoisRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { + if ((now - _lastWhoisRequestReceived) >= ZT_PEER_WHOIS_RATE_LIMIT) { _lastWhoisRequestReceived = now; return true; } diff --git a/node/Switch.cpp b/node/Switch.cpp index 0392aec1..85103aa5 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -688,9 +688,9 @@ unsigned long Switch::doTimerTasks(uint64_t now) _outstandingWhoisRequests.erase(*a); } else { r->lastSent = now; - r->peersConsulted[r->retries] = _sendWhoisRequest(*a,r->peersConsulted,r->retries); - ++r->retries; + r->peersConsulted[r->retries] = _sendWhoisRequest(*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); TRACE("WHOIS %s (retry %u)",a->toString().c_str(),r->retries); + ++r->retries; nextDelay = std::min(nextDelay,(unsigned long)ZT_WHOIS_RETRY_DELAY); } } else { -- cgit v1.2.3 From 0f3148bda2420c99529194ccc1422d22bd6575bd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 10 Mar 2017 20:08:07 -0800 Subject: Roots need to respond to lots of WHOISes --- node/Constants.hpp | 2 +- node/IncomingPacket.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index c2961f12..410a245b 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -393,7 +393,7 @@ /** * WHOIS rate limit (we allow these to be pretty fast) */ -#define ZT_PEER_WHOIS_RATE_LIMIT 50 +#define ZT_PEER_WHOIS_RATE_LIMIT 100 /** * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index f3ec7505..800985dc 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -552,7 +552,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer) { try { - if (!peer->rateGateInboundWhoisRequest(RR->node->now())) { + if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) { TRACE("dropped WHOIS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); return true; } -- cgit v1.2.3 From 010d0a7d569e3aab5261c68e4530e82171b2e311 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 13 Mar 2017 06:53:23 -0700 Subject: Docs and a bit of cleanup. In particular ALL makes no sense for revocations because they have IDs. In that case you would just revoke the COM. --- node/Capability.hpp | 5 +++++ node/CertificateOfRepresentation.hpp | 15 +++++++++++++++ node/Membership.cpp | 3 +-- node/Revocation.hpp | 14 +++++++++++++- 4 files changed, 34 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index 1ad6ea42..d070f2ad 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -52,6 +52,11 @@ class RuntimeEnvironment; * * Note that this is after evaluation of network scope rules and only if * network scope rules do not deliver an explicit match. + * + * Capabilities support a chain of custody. This is currently unused but + * in the future would allow the publication of capabilities that can be + * handed off between nodes. Limited transferrability of capabilities is + * a feature of true capability based security. */ class Capability { diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp index 7c239a96..02e961c4 100644 --- a/node/CertificateOfRepresentation.hpp +++ b/node/CertificateOfRepresentation.hpp @@ -32,6 +32,21 @@ namespace ZeroTier { +/** + * A signed enumeration of a node's roots (planet and moons) + * + * This is sent as part of HELLO and attests to which roots a node trusts + * to represent it on the network. Federated roots (moons) can send these + * further upstream to tell global roots which nodes they represent, making + * them reachable via federated roots if they are not reachable directly. + * + * As of 1.2.0 this is sent but not used. Right now nodes still always + * announce to planetary roots no matter what. In the future this can be + * used to implement even better fault tolerance for federation for the + * no roots are reachable case as well as a "privacy mode" where federated + * roots can shield nodes entirely and p2p connectivity behind them can + * be disabled. This will be desirable for a number of use cases. + */ class CertificateOfRepresentation { public: diff --git a/node/Membership.cpp b/node/Membership.cpp index a60b86be..3b2e3b1c 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -223,8 +223,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme const uint64_t now = RR->node->now(); switch(rev.type()) { default: - //case Revocation::CREDENTIAL_TYPE_ALL: - return ( (_revokeCom(rev)||_revokeCap(rev,now)||_revokeTag(rev,now)||_revokeCoo(rev,now)) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT ); + return ADD_REJECTED; case Revocation::CREDENTIAL_TYPE_COM: return (_revokeCom(rev) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); case Revocation::CREDENTIAL_TYPE_CAPABILITY: diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 1697b52f..93c55112 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -47,9 +47,12 @@ class RuntimeEnvironment; class Revocation { public: + /** + * Credential type being revoked + */ enum CredentialType { - CREDENTIAL_TYPE_ALL = 0, + CREDENTIAL_TYPE_NULL = 0, CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership CREDENTIAL_TYPE_CAPABILITY = 2, CREDENTIAL_TYPE_TAG = 3, @@ -61,6 +64,15 @@ public: memset(this,0,sizeof(Revocation)); } + /** + * @param i ID (arbitrary for revocations, currently random) + * @param nwid Network ID + * @param cid Credential ID being revoked (0 for all or for COMs, which lack IDs) + * @param thr Revocation time threshold before which credentials will be revoked + * @param fl Flags + * @param tgt Target node whose credential(s) are being revoked + * @param ct Credential type being revoked + */ Revocation(const uint64_t i,const uint64_t nwid,const uint64_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const CredentialType ct) : _id(i), _networkId(nwid), -- cgit v1.2.3 From c6a39ed927161736e44aeaa67c6783024c1fb86a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 13:55:26 -0700 Subject: Fixes for possible ARM issues, cleanup, fix for spurious meaningless exceptions on NETWORK_CONFIG_REQUEST --- make-linux.mk | 14 +++++++- node/Dictionary.hpp | 25 +++++++++++---- node/IncomingPacket.cpp | 6 ++-- node/Network.cpp | 8 +++++ one.cpp | 85 +++++++++++++++++++++++++++++++------------------ service/OneService.cpp | 2 ++ 6 files changed, 99 insertions(+), 41 deletions(-) (limited to 'node') diff --git a/make-linux.mk b/make-linux.mk index 1bb62852..528c41c4 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -54,7 +54,7 @@ ifeq ($(ZT_RULES_ENGINE_DEBUGGING),1) endif ifeq ($(ZT_DEBUG),1) - DEFS+=-DZT_TRACE +# DEFS+=-DZT_TRACE override CFLAGS+=-Wall -g -O -pthread $(INCLUDES) $(DEFS) override CXXFLAGS+=-Wall -g -O -std=c++11 -pthread $(INCLUDES) $(DEFS) override LDFLAGS+= @@ -96,6 +96,12 @@ endif ifeq ($(CC_MACH),arm) ZT_ARCHITECTURE=3 endif +ifeq ($(CC_MACH),armv6) + ZT_ARCHITECTURE=3 +endif +ifeq ($(CC_MACH),armv7) + ZT_ARCHITECTURE=3 +endif ifeq ($(CC_MACH),arm64) ZT_ARCHITECTURE=4 endif @@ -104,6 +110,12 @@ ifeq ($(CC_MACH),aarch64) endif DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" +# Define some conservative CPU instruction set flags for arm32 since there's a ton of variation out there +ifeq ($(ZT_ARCHITECTURE),3) + override CFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfloat-abi=hard -mfpu=vfp + override CXXFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfloat-abi=hard -mfpu=vfp +endif + # Define this to build a static binary, which is needed to make this runnable on a few ancient Linux distros ifeq ($(ZT_STATIC),1) override LDFLAGS+=-static diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 15ab9ce3..fa9e2883 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -61,15 +61,23 @@ public: Dictionary(const char *s) { - Utils::scopy(_d,sizeof(_d),s); + if (s) { + Utils::scopy(_d,sizeof(_d),s); + } else { + _d[0] = (char)0; + } } Dictionary(const char *s,unsigned int len) { - if (len > (C-1)) - len = C-1; - memcpy(_d,s,len); - _d[len] = (char)0; + if (s) { + if (len > (C-1)) + len = C-1; + memcpy(_d,s,len); + _d[len] = (char)0; + } else { + _d[0] = (char)0; + } } Dictionary(const Dictionary &d) @@ -91,7 +99,12 @@ public: */ inline bool load(const char *s) { - return Utils::scopy(_d,sizeof(_d),s); + if (s) { + return Utils::scopy(_d,sizeof(_d),s); + } else { + _d[0] = (char)0; + return true; + } } /** diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 800985dc..ac4ae377 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -836,7 +836,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S bool trustEstablished = false; unsigned int p = ZT_PACKET_IDX_PAYLOAD; - while ((p < size())&&((*this)[p])) { + while ((p < size())&&((*this)[p] != 0)) { p += com.deserialize(*this,p); if (com) { const SharedPtr network(RR->node->network(com.networkId())); @@ -953,8 +953,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons const uint64_t requestPacketId = packetId(); if (RR->localNetworkController) { - const unsigned int metaDataLength = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN); - const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength); + const unsigned int metaDataLength = (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN <= size()) ? at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN) : 0; + const char *metaDataBytes = (metaDataLength != 0) ? (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength) : (const char *)0; const Dictionary metaData(metaDataBytes,metaDataLength); RR->localNetworkController->request(nwid,(hopCount > 0) ? InetAddress() : _path->address(),requestPacketId,peer->identity(),metaData); } else { diff --git a/node/Network.cpp b/node/Network.cpp index dd812cab..92ca67ab 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1517,6 +1517,10 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); _config.com.serialize(outp); outp.append((uint8_t)0x00); + outp.append((uint16_t)0); // no capabilities + outp.append((uint16_t)0); // no tags + outp.append((uint16_t)0); // no revocations + outp.append((uint16_t)0); // no certificates of ownership RR->sw->send(outp,true); } _announceMulticastGroupsTo(*a,groups); @@ -1529,6 +1533,10 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Packet outp(c,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); _config.com.serialize(outp); outp.append((uint8_t)0x00); + outp.append((uint16_t)0); // no capabilities + outp.append((uint16_t)0); // no tags + outp.append((uint16_t)0); // no revocations + outp.append((uint16_t)0); // no certificates of ownership RR->sw->send(outp,true); } _announceMulticastGroupsTo(c,groups); diff --git a/one.cpp b/one.cpp index 95230bf1..25a50dbb 100644 --- a/one.cpp +++ b/one.cpp @@ -72,6 +72,7 @@ #include "osdep/OSUtils.hpp" #include "osdep/Http.hpp" +#include "osdep/Thread.hpp" #include "service/OneService.hpp" @@ -1209,6 +1210,52 @@ static void printHelp(const char *cn,FILE *out) fprintf(out," -q - Query API (zerotier-cli)" ZT_EOL_S); } +class _OneServiceRunner +{ +public: + _OneServiceRunner(const char *pn,const std::string &hd,unsigned int p) : progname(pn),returnValue(0),port(p),homeDir(hd) {} + void threadMain() + throw() + { + try { + for(;;) { + zt1Service = OneService::newInstance(homeDir.c_str(),port); + switch(zt1Service->run()) { + case OneService::ONE_STILL_RUNNING: // shouldn't happen, run() won't return until done + case OneService::ONE_NORMAL_TERMINATION: + break; + case OneService::ONE_UNRECOVERABLE_ERROR: + fprintf(stderr,"%s: fatal error: %s" ZT_EOL_S,progname,zt1Service->fatalErrorMessage().c_str()); + returnValue = 1; + break; + case OneService::ONE_IDENTITY_COLLISION: { + delete zt1Service; + zt1Service = (OneService *)0; + std::string oldid; + OSUtils::readFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str(),oldid); + if (oldid.length()) { + OSUtils::writeFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret.saved_after_collision").c_str(),oldid); + OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str()); + OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.public").c_str()); + } + } continue; // restart! + } + break; // terminate loop -- normally we don't keep restarting + } + + delete zt1Service; + zt1Service = (OneService *)0; + } catch ( ... ) { + fprintf(stderr,"%s: unexpected exception starting main OneService instance" ZT_EOL_S,progname); + returnValue = 1; + } + } + const char *progname; + unsigned int returnValue; + unsigned int port; + const std::string &homeDir; +}; + #ifdef __WINDOWS__ int _tmain(int argc, _TCHAR* argv[]) #else @@ -1421,8 +1468,8 @@ int main(int argc,char **argv) } else { // Running from service manager _winPokeAHole(); - ZeroTierOneService zt1Service; - if (CServiceBase::Run(zt1Service) == TRUE) { + ZeroTierOneService zt1WindowsService; + if (CServiceBase::Run(zt1WindowsService) == TRUE) { return 0; } else { fprintf(stderr,"%s: unable to start service (try -h for help)" ZT_EOL_S,argv[0]); @@ -1448,35 +1495,11 @@ int main(int argc,char **argv) } #endif // __UNIX_LIKE__ - unsigned int returnValue = 0; - - for(;;) { - zt1Service = OneService::newInstance(homeDir.c_str(),port); - switch(zt1Service->run()) { - case OneService::ONE_STILL_RUNNING: // shouldn't happen, run() won't return until done - case OneService::ONE_NORMAL_TERMINATION: - break; - case OneService::ONE_UNRECOVERABLE_ERROR: - fprintf(stderr,"%s: fatal error: %s" ZT_EOL_S,argv[0],zt1Service->fatalErrorMessage().c_str()); - returnValue = 1; - break; - case OneService::ONE_IDENTITY_COLLISION: { - delete zt1Service; - zt1Service = (OneService *)0; - std::string oldid; - OSUtils::readFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str(),oldid); - if (oldid.length()) { - OSUtils::writeFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret.saved_after_collision").c_str(),oldid); - OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str()); - OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.public").c_str()); - } - } continue; // restart! - } - break; // terminate loop -- normally we don't keep restarting - } + _OneServiceRunner thr(argv[0],homeDir,port); + thr.threadMain(); + //Thread::join(Thread::start(&thr)); - delete zt1Service; - zt1Service = (OneService *)0; + OSUtils::rm(pidPath.c_str()); - return returnValue; + return thr.returnValue; } diff --git a/service/OneService.cpp b/service/OneService.cpp index 4a2102f1..1c2fa05d 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -488,6 +488,8 @@ public: ,_updater((SoftwareUpdater *)0) ,_updateAutoApply(false) ,_primaryPort(port) + ,_v4TcpControlSocket((PhySocket *)0) + ,_v6TcpControlSocket((PhySocket *)0) ,_lastDirectReceiveFromGlobal(0) #ifdef ZT_TCP_FALLBACK_RELAY ,_lastSendToGlobalV4(0) -- cgit v1.2.3 From d1bb22a583883a86cde9c845afb3e8b884f6301d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 14:09:30 -0700 Subject: . --- node/IncomingPacket.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ac4ae377..dc6140f5 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -938,9 +938,11 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); } catch (std::exception &exc) { + fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { - TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); + TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } @@ -969,10 +971,10 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); } catch (std::exception &exc) { - fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what()); + fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { - fprintf(stderr,"WARNING: network config request failed with exception: unknown exception" ZT_EOL_S); + fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; -- cgit v1.2.3 From a7cb738175cde9ba85143ae7076ba6b18bcc1c5b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 14:25:54 -0700 Subject: . --- node/IncomingPacket.cpp | 8 ++++---- node/Network.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index dc6140f5..2e4a0b8e 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -938,10 +938,10 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); } catch (std::exception &exc) { - fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); + //fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { - fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); + //fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): unknown exception" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str()); TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; @@ -971,10 +971,10 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); } catch (std::exception &exc) { - fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); + //fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { - fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); + //fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; diff --git a/node/Network.cpp b/node/Network.cpp index 92ca67ab..38c1b0d9 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1282,8 +1282,8 @@ void Network::requestConfiguration() } else { outp.append((unsigned char)0,16); } - RR->node->expectReplyTo(outp.packetId()); outp.compress(); + RR->node->expectReplyTo(outp.packetId()); RR->sw->send(outp,true); } -- cgit v1.2.3 From cdc0eaec3add50e1424a0bcd9d054ec140c3540b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 22:13:34 +0000 Subject: Fix attempt to WHOIS self. --- .gitignore | 1 + make-linux.mk | 2 +- node/IncomingPacket.cpp | 6 +++++- node/Switch.cpp | 7 +++++++ node/Topology.hpp | 20 ++++++++++++-------- 5 files changed, 26 insertions(+), 10 deletions(-) (limited to 'node') diff --git a/.gitignore b/.gitignore index 8d404eef..437352a3 100755 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .DS_Store .Apple* Thumbs.db +@eaDir # Windows build droppings /windows/ZeroTierOne.sdf diff --git a/make-linux.mk b/make-linux.mk index 528c41c4..17bcd158 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -54,7 +54,7 @@ ifeq ($(ZT_RULES_ENGINE_DEBUGGING),1) endif ifeq ($(ZT_DEBUG),1) -# DEFS+=-DZT_TRACE + DEFS+=-DZT_TRACE override CFLAGS+=-Wall -g -O -pthread $(INCLUDES) $(DEFS) override CXXFLAGS+=-Wall -g -O -std=c++11 -pthread $(INCLUDES) $(DEFS) override LDFLAGS+= diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 2e4a0b8e..dc2c8aaf 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -477,7 +477,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p } else ptr += 2; } - TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)")); +#ifdef ZT_TRACE + const std::string tmp1(source().toString()); + const std::string tmp2(_path->address().toString()); + TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u",tmp1.c_str(),tmp2.c_str(),vMajor,vMinor,vRevision,latency); +#endif if (!hops()) peer->addDirectLatencyMeasurment((unsigned int)latency); diff --git a/node/Switch.cpp b/node/Switch.cpp index 85103aa5..aab2e7ff 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -625,6 +625,13 @@ void Switch::send(Packet &packet,bool encrypt) void Switch::requestWhois(const Address &addr) { +#ifdef ZT_TRACE + if (addr == RR->identity.address()) { + fprintf(stderr,"FATAL BUG: Switch::requestWhois() caught attempt to WHOIS self" ZT_EOL_S); + abort(); + } +#endif + bool inserted = false; { Mutex::Lock _l(_outstandingWhoisRequests_m); diff --git a/node/Topology.hpp b/node/Topology.hpp index 37615b49..e21747c8 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -184,14 +184,7 @@ public: { Mutex::Lock _l(_upstreams_m); for(std::vector::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) { - std::vector &ips = eps[i->identity.address()]; - for(std::vector::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) { - if (std::find(ips.begin(),ips.end(),*j) == ips.end()) - ips.push_back(*j); - } - } - for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { - for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { + if (i->identity != RR->identity) { std::vector &ips = eps[i->identity.address()]; for(std::vector::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) { if (std::find(ips.begin(),ips.end(),*j) == ips.end()) @@ -199,6 +192,17 @@ public: } } } + for(std::vector::const_iterator m(_moons.begin());m!=_moons.end();++m) { + for(std::vector::const_iterator i(m->roots().begin());i!=m->roots().end();++i) { + if (i->identity != RR->identity) { + std::vector &ips = eps[i->identity.address()]; + for(std::vector::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) { + if (std::find(ips.begin(),ips.end(),*j) == ips.end()) + ips.push_back(*j); + } + } + } + } for(std::vector< std::pair >::const_iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m) eps[m->second]; } -- cgit v1.2.3 From c467c3b7e4f54b09352cd3f96b78b21fe963aace Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 22:26:08 +0000 Subject: ARM tweaks --- make-linux.mk | 7 +++++++ node/Packet.cpp | 8 ++++++++ 2 files changed, 15 insertions(+) (limited to 'node') diff --git a/make-linux.mk b/make-linux.mk index 17bcd158..7c77f58f 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -96,6 +96,12 @@ endif ifeq ($(CC_MACH),arm) ZT_ARCHITECTURE=3 endif +ifeq ($(CC_MACH),armel) + ZT_ARCHITECTURE=3 +endif +ifeq ($(CC_MACH),armhf) + ZT_ARCHITECTURE=3 +endif ifeq ($(CC_MACH),armv6) ZT_ARCHITECTURE=3 endif @@ -114,6 +120,7 @@ DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFT ifeq ($(ZT_ARCHITECTURE),3) override CFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfloat-abi=hard -mfpu=vfp override CXXFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfloat-abi=hard -mfpu=vfp + override DEFS+=-DZT_NO_TYPE_PUNNING endif # Define this to build a static binary, which is needed to make this runnable on a few ancient Linux distros diff --git a/node/Packet.cpp b/node/Packet.cpp index c825ea9b..6dfba9f3 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -520,6 +520,7 @@ union LZ4_streamDecode_u { * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) */ +#if 0 #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define LZ4_FORCE_MEMORY_ACCESS 2 @@ -528,6 +529,13 @@ union LZ4_streamDecode_u { # define LZ4_FORCE_MEMORY_ACCESS 1 # endif #endif +#endif + +#ifdef ZT_NO_TYPE_PUNNING +#define LZ4_FORCE_MEMORY_ACCESS 0 +#else +#define LZ4_FORCE_MEMORY_ACCESS 2 +#endif /* * LZ4_FORCE_SW_BITCOUNT -- cgit v1.2.3 From a9c08c5975691b74d8e6e81c95bdd85a7a271c39 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 22:35:56 +0000 Subject: . --- node/Packet.cpp | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index 6dfba9f3..d5b3d720 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -122,6 +122,7 @@ namespace { * LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL */ +#if 0 #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) # define LZ4LIB_API __declspec(dllexport) #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) @@ -129,7 +130,9 @@ namespace { #else # define LZ4LIB_API #endif - +#else +#define LZ4LIB_API +#endif /*========== Version =========== */ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ @@ -378,9 +381,6 @@ LZ4_decompress_*_continue() : #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -//#include - typedef struct { uint32_t hashTable[LZ4_HASH_SIZE_U32]; uint32_t currentOffset; @@ -397,26 +397,6 @@ typedef struct { size_t prefixSize; } LZ4_streamDecode_t_internal; -#else - -typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#endif - /*! * LZ4_stream_t : * information structure to track an LZ4 stream. @@ -2046,7 +2026,7 @@ bool Packet::compress() unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) { int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); + int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,1); if ((cl > 0)&&(cl < pl)) { (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); -- cgit v1.2.3 From ef46d3c97dd5d63af7b638f6107bb56495bab9a5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 23:09:18 +0000 Subject: LZ4 cleanup --- node/Packet.cpp | 959 ++------------------------------------------------------ 1 file changed, 30 insertions(+), 929 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index d5b3d720..eb866568 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -89,9 +89,6 @@ namespace { - LZ4 source repository : https://github.com/lz4/lz4 */ -/* --- Dependency --- */ -//#include /* size_t */ - /** Introduction @@ -115,24 +112,7 @@ namespace { A library is provided to take care of it, see lz4frame.h. */ -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if 0 -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LZ4LIB_API -#endif -#else #define LZ4LIB_API -#endif /*========== Version =========== */ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ @@ -146,13 +126,6 @@ namespace { #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) -//LZ4LIB_API int LZ4_versionNumber (void); -//LZ4LIB_API const char* LZ4_versionString (void); - - -/*-************************************ -* Tuning parameter -**************************************/ /*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) @@ -162,211 +135,20 @@ namespace { */ #define LZ4_MEMORY_USAGE 14 - -/*-************************************ -* Simple Functions -**************************************/ -/*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -//LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); - -/*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - - -/*-************************************ -* Advanced Functions -**************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -/*! -LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) - inputSize : max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) -*/ -LZ4LIB_API int LZ4_compressBound(int inputSize); - -/*! -LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. - The larger the acceleration value, the faster the algorithm, but also the lesser the compression. - It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. -*/ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ -//LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails -*/ -//LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); - - -/*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -//LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -//LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); - - /*-********************************************* * Streaming Compression Functions ***********************************************/ typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ -//LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); -//LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); - /*! LZ4_resetStream() : * An LZ4_stream_t structure can be allocated once and re-used multiple times. * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. */ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -/*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= 64 KB) - */ -//LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); - -/*! LZ4_compress_fast_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to still be present and unmodified ! - * 'dst' buffer must be already allocated. - * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - */ -//LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -//LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); - - -/*-********************************************** -* Streaming Decompression Functions -* Bufferless synchronous API -************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ - -/* creation / destruction of streaming decompression tracking structure */ -//LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); -//LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - -/*! LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * @return : 1 if OK, 0 if error - */ -//LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); - -/*! -LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() -*/ -//LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -//LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); - - -/*! LZ4_decompress_*_usingDict() : - * These decoding functions work the same as - * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() - * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - */ -//LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -//LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); - - /*^********************************************** * !!!!!! STATIC LINKING ONLY !!!!!! ***********************************************/ @@ -412,7 +194,6 @@ union LZ4_stream_u { LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ - /*! * LZ4_streamDecode_t : * information structure to track an LZ4 stream during decompression. @@ -428,65 +209,12 @@ union LZ4_streamDecode_u { LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ -/* lz4.c ------------------------------------------------------------------ */ - -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - - -/*-************************************ -* Tuning parameters -**************************************/ -/* - * HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). - */ #ifndef HEAPMODE # define HEAPMODE 0 #endif -/* - * ACCELERATION_DEFAULT : - * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 - */ -#define ACCELERATION_DEFAULT 1 - +//#define ACCELERATION_DEFAULT 1 -/*-************************************ -* CPU Feature Detection -**************************************/ /* LZ4_FORCE_MEMORY_ACCESS * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. @@ -525,14 +253,6 @@ union LZ4_streamDecode_u { # define LZ4_FORCE_SW_BITCOUNT #endif - -/*-************************************ -* Dependency -**************************************/ -//#include "lz4.h" -/* see also "memory routines" below */ - - /*-************************************ * Compiler Options **************************************/ @@ -553,6 +273,8 @@ union LZ4_streamDecode_u { #endif /* _MSC_VER */ #endif +#define FORCE_INLINE static inline + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else @@ -562,7 +284,6 @@ union LZ4_streamDecode_u { #define likely(expr) expect((expr) != 0, 1) #define unlikely(expr) expect((expr) != 0, 0) - /*-************************************ * Memory routines **************************************/ @@ -572,33 +293,16 @@ union LZ4_streamDecode_u { //#include /* memset, memcpy */ #define MEM_INIT memset - /*-************************************ * Basic Types **************************************/ -//#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -//# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef uintptr_t uptrval; -/*#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef size_t uptrval; -#endif */ - +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +typedef uintptr_t uptrval; typedef uintptr_t reg_t; -//#if defined(__x86_64__) -// typedef U64 reg_t; /* 64-bits in x32 mode */ -//#else -// typedef size_t reg_t; /* 32-bits in x32 mode */ -//#endif /*-************************************ * Reading and writing into memory @@ -634,35 +338,34 @@ static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = val #else /* safe and portable access through memcpy() */ -static U16 LZ4_read16(const void* memPtr) +static inline U16 LZ4_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; } -static U32 LZ4_read32(const void* memPtr) +static inline U32 LZ4_read32(const void* memPtr) { U32 val; memcpy(&val, memPtr, sizeof(val)); return val; } -static reg_t LZ4_read_ARCH(const void* memPtr) +static inline reg_t LZ4_read_ARCH(const void* memPtr) { reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; } -static void LZ4_write16(void* memPtr, U16 value) +static inline void LZ4_write16(void* memPtr, U16 value) { memcpy(memPtr, &value, sizeof(value)); } -static void LZ4_write32(void* memPtr, U32 value) +static inline void LZ4_write32(void* memPtr, U32 value) { memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ - -static U16 LZ4_readLE16(const void* memPtr) +static inline U16 LZ4_readLE16(const void* memPtr) { if (LZ4_isLittleEndian()) { return LZ4_read16(memPtr); @@ -672,7 +375,7 @@ static U16 LZ4_readLE16(const void* memPtr) } } -static void LZ4_writeLE16(void* memPtr, U16 value) +static inline void LZ4_writeLE16(void* memPtr, U16 value) { if (LZ4_isLittleEndian()) { LZ4_write16(memPtr, value); @@ -683,13 +386,13 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } } -static void LZ4_copy8(void* dst, const void* src) +static inline void LZ4_copy8(void* dst, const void* src) { memcpy(dst,src,8); } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +static inline void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; @@ -698,7 +401,6 @@ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) do { LZ4_copy8(d,s); d+=8; s+=8; } while (d compression run slower on incompressible data */ - /*-************************************ * Local Structures and types **************************************/ @@ -828,7 +525,6 @@ typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { full = 0, partial = 1 } earlyEnd_directive; - /*-************************************ * Local Utils **************************************/ @@ -837,11 +533,10 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } //int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +static inline U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -849,7 +544,7 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +static inline U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { static const U64 prime5bytes = 889523592379ULL; static const U64 prime8bytes = 11400714785074694791ULL; @@ -866,7 +561,7 @@ FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableTy return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +static inline void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { switch (tableType) { @@ -882,7 +577,7 @@ FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t ta LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +static inline const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } @@ -895,7 +590,6 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } - /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ FORCE_INLINE int LZ4_compress_generic( @@ -1097,12 +791,11 @@ _last_literals: return (int) (((char*)op)-dest); } - -int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +static inline int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + //if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) @@ -1117,8 +810,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int } } - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +static inline int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { #if (HEAPMODE) void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ @@ -1135,387 +827,11 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp return result; } -#if 0 -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} -#endif - -/*-****************************** -* *_destSize() variant -********************************/ - -#if 0 -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } -} - -int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) -{ -#if (HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; -#endif - - int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); - -#if (HEAPMODE) - FREEMEM(ctx); -#endif - return result; -} -#endif - -/*-****************************** -* Streaming functions -********************************/ - -#if 0 -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; -} -#endif - void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } -#if 0 -int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return (0); -} -#endif - -#if 0 -#define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) -{ - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; - - while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); - p+=3; - } - - return dict->dictSize; -} - -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) -{ - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } -} - -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - /* external dictionary mode */ - { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); - - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - - return result; -} - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -{ - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; - - return dictSize; -} - -#endif - /*-***************************** * Decompression functions *******************************/ @@ -1686,226 +1002,11 @@ _output_error: return (int) (-(((const char*)ip)-source))-1; } - -int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +static inline int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); } -#if 0 -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); -} - -int LZ4_decompress_fast(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); -} -#endif - -/*===== streaming decompression functions =====*/ - -#if 0 -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - */ -LZ4_streamDecode_t* LZ4_createStreamDecode(void) -{ - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); - return lz4s; -} - -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return 0; -} - -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error - */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() -*/ -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); -} - -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -#endif - -#if 0 -/*=************************************************* -* Obsolete Functions -***************************************************/ -/* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } - -/* -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe -*/ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} - -int LZ4_resetStreamState(void* state, char* inputBuffer) -{ - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); - return 0; -} - -void* LZ4_create (char* inputBuffer) -{ - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; -} - -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} -#endif - -#endif /* LZ4_COMMONDEFS_ONLY */ - } // anonymous namespace /************************************************************************** */ @@ -2024,9 +1125,9 @@ void Packet::cryptField(const void *key,unsigned int start,unsigned int len) bool Packet::compress() { unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; - if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) { + if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 64))) { // don't bother compressing tiny packets int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,1); + int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); if ((cl > 0)&&(cl < pl)) { (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); -- cgit v1.2.3 From e10325e133beb01a2e9d82687eb33fd72f1ac3ab Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 17:15:23 -0700 Subject: GitHub issue #461 -- plus a bit of cleanup and optimization --- node/CertificateOfOwnership.hpp | 7 ++----- node/IncomingPacket.cpp | 4 +++- node/Network.hpp | 1 + node/OutboundMulticast.cpp | 4 +++- node/Packet.cpp | 30 +++++++++++++++++++----------- node/Packet.hpp | 2 +- 6 files changed, 29 insertions(+), 19 deletions(-) (limited to 'node') diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 7e71c9b2..57fd8259 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -56,12 +56,9 @@ public: THING_IPV6_ADDRESS = 3 }; - CertificateOfOwnership() : - _networkId(0), - _ts(0), - _id(0), - _thingCount(0) + CertificateOfOwnership() { + memset(this,0,sizeof(CertificateOfOwnership)); } CertificateOfOwnership(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id) : diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index dc2c8aaf..e2275a04 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -72,13 +72,15 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) if (peer) { if (!trusted) { if (!dearmor(peer->key())) { + //fprintf(stderr,"dropped packet from %s(%s), MAC authentication failed (size: %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); TRACE("dropped packet from %s(%s), MAC authentication failed (size: %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); return true; } } if (!uncompress()) { - TRACE("dropped packet from %s(%s), compressed data invalid (verb may be %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),(unsigned int)verb()); + //fprintf(stderr,"dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); + TRACE("dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); return true; } diff --git a/node/Network.hpp b/node/Network.hpp index 56c7fc60..6cf6d974 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -374,6 +374,7 @@ private: struct _IncomingConfigChunk { + _IncomingConfigChunk() { memset(this,0,sizeof(_IncomingConfigChunk)); } uint64_t ts; uint64_t updateId; uint64_t haveChunkIds[ZT_NETWORK_MAX_UPDATE_CHUNKS]; diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 36dc41f4..d4cb87cb 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -94,7 +94,9 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toA _packet.newInitializationVector(); _packet.setDestination(toAddr2); RR->node->expectReplyTo(_packet.packetId()); - RR->sw->send(_packet,true); + + Packet tmp(_packet); // make a copy of packet so as not to garble the original -- GitHub issue #461 + RR->sw->send(tmp,true); } } diff --git a/node/Packet.cpp b/node/Packet.cpp index eb866568..b07f0bed 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1066,7 +1066,7 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) uint8_t *const data = reinterpret_cast(unsafeData()); // Mask least significant 3 bits of packet ID with counter to embed packet send counter for QoS use - data[7] = (data[7] & 0xf8) | ((uint8_t)counter & 0x07); + data[7] = (data[7] & 0xf8) | (uint8_t)(counter & 0x07); // Set flag now, since it affects key mangle function setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE); @@ -1124,35 +1124,43 @@ void Packet::cryptField(const void *key,unsigned int start,unsigned int len) bool Packet::compress() { - unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; + char *const data = reinterpret_cast(unsafeData()); + char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; + if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 64))) { // don't bother compressing tiny packets int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); + int cl = LZ4_compress_fast(data + ZT_PACKET_IDX_PAYLOAD,buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); if ((cl > 0)&&(cl < pl)) { - (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; + data[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); - memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)cl),buf,cl); + memcpy(data + ZT_PACKET_IDX_PAYLOAD,buf,cl); return true; } } - (*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED); + data[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED); + return false; } bool Packet::uncompress() { - unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH]; + char *const data = reinterpret_cast(unsafeData()); + char buf[ZT_PROTO_MAX_PACKET_LENGTH]; + if ((compressed())&&(size() >= ZT_PROTO_MIN_PACKET_LENGTH)) { if (size() > ZT_PACKET_IDX_PAYLOAD) { unsigned int compLen = size() - ZT_PACKET_IDX_PAYLOAD; - int ucl = LZ4_decompress_safe((const char *)field(ZT_PACKET_IDX_PAYLOAD,compLen),(char *)buf,compLen,sizeof(buf)); + int ucl = LZ4_decompress_safe((const char *)data + ZT_PACKET_IDX_PAYLOAD,buf,compLen,sizeof(buf)); if ((ucl > 0)&&(ucl <= (int)(capacity() - ZT_PACKET_IDX_PAYLOAD))) { setSize((unsigned int)ucl + ZT_PACKET_IDX_PAYLOAD); - memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)ucl),buf,ucl); - } else return false; + memcpy(data + ZT_PACKET_IDX_PAYLOAD,buf,ucl); + } else { + return false; + } } - (*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED); + data[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED); } + return true; } diff --git a/node/Packet.hpp b/node/Packet.hpp index fb332b7d..8ad2c0f9 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1322,7 +1322,7 @@ public: /** * @return Value of link quality counter extracted from this packet's ID, range 0 to 7 (3 bits) */ - inline unsigned int linkQualityCounter() const { return (unsigned int)(reinterpret_cast(data())[7] & 7); } + inline unsigned int linkQualityCounter() const { return (unsigned int)(reinterpret_cast(data())[7] & 0x07); } /** * Set packet verb -- cgit v1.2.3 From 78ef2c5f16524684b4682ac2ef614abe1ed62dd7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 17 Mar 2017 20:01:58 -0700 Subject: Windows build fixes, app about text revisions. --- node/Packet.cpp | 2 + one.cpp | 3 +- osdep/OSUtils.cpp | 2 +- service/SoftwareUpdater.cpp | 4 +- windows/WinUI/AboutView.xaml | 120 ++++++++++--------------------------------- 5 files changed, 33 insertions(+), 98 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index b07f0bed..756f3140 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -273,7 +273,9 @@ union LZ4_streamDecode_u { #endif /* _MSC_VER */ #endif +#ifndef FORCE_INLINE #define FORCE_INLINE static inline +#endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) diff --git a/one.cpp b/one.cpp index 25a50dbb..edefe82e 100644 --- a/one.cpp +++ b/one.cpp @@ -1479,7 +1479,6 @@ int main(int argc,char **argv) #endif // __WINDOWS__ #ifdef __UNIX_LIKE__ - #ifdef ZT_HAVE_DROP_PRIVILEGES dropPrivileges(argv[0],homeDir); #endif @@ -1499,7 +1498,9 @@ int main(int argc,char **argv) thr.threadMain(); //Thread::join(Thread::start(&thr)); +#ifdef __UNIX_LIKE__ OSUtils::rm(pidPath.c_str()); +#endif return thr.returnValue; } diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index aac6bdd8..33e143da 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -125,7 +125,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) date.LowPart = ffd.ftLastWriteTime.dwLowDateTime; if (date.QuadPart > 0) { date.QuadPart -= adjust.QuadPart; - if (((date.QuadPart / 10000000) * 1000) < olderThan) { + if ((uint64_t)((date.QuadPart / 10000000) * 1000) < olderThan) { Utils::snprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); if (DeleteFileA(tmp)) ++cleaned; diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 2afbc776..7ecd42b1 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -148,9 +148,9 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void std::vector dvArch; if (dvArch2.is_array()) { for(unsigned long i=0;isecond.meta[ZT_SOFTWARE_UPDATE_JSON_PLATFORM],0) == rvPlatform)&& diff --git a/windows/WinUI/AboutView.xaml b/windows/WinUI/AboutView.xaml index 5def46a6..7f4151f9 100644 --- a/windows/WinUI/AboutView.xaml +++ b/windows/WinUI/AboutView.xaml @@ -5,99 +5,31 @@ xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:local="clr-namespace:WinUI" mc:Ignorable="d" - Title="AboutView" Height="460" Width="300" Icon="ZeroTierIcon.ico"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Title="AboutView" Height="368.267" Width="300" Icon="ZeroTierIcon.ico"> + + + + + + + + + + + + + + + + + + + + + + - + -- cgit v1.2.3 From e4896b257fde05a216500804d9bcef3b84b0980e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 27 Mar 2017 17:03:17 -0700 Subject: Add thread PTR that gets passed through the entire ZT core call stack and then passed to handler functions resulting from a call. --- controller/EmbeddedNetworkController.cpp | 2 +- include/ZeroTierOne.h | 51 ++++-- node/Capability.cpp | 6 +- node/Capability.hpp | 2 +- node/CertificateOfMembership.cpp | 6 +- node/CertificateOfMembership.hpp | 3 +- node/CertificateOfOwnership.cpp | 6 +- node/CertificateOfOwnership.hpp | 3 +- node/IncomingPacket.cpp | 294 +++++++++++++++---------------- node/IncomingPacket.hpp | 43 ++--- node/Membership.cpp | 24 +-- node/Membership.hpp | 13 +- node/Multicaster.cpp | 23 +-- node/Multicaster.hpp | 13 +- node/Network.cpp | 112 ++++++------ node/Network.hpp | 59 ++++--- node/Node.cpp | 136 +++++++------- node/Node.hpp | 42 +++-- node/OutboundMulticast.cpp | 6 +- node/OutboundMulticast.hpp | 13 +- node/Path.cpp | 4 +- node/Path.hpp | 3 +- node/Peer.cpp | 37 ++-- node/Peer.hpp | 22 ++- node/Revocation.cpp | 6 +- node/Revocation.hpp | 3 +- node/SelfAwareness.cpp | 10 +- node/SelfAwareness.hpp | 2 +- node/Switch.cpp | 97 +++++----- node/Switch.hpp | 22 ++- node/Tag.cpp | 6 +- node/Tag.hpp | 3 +- node/Topology.cpp | 54 +++--- node/Topology.hpp | 26 +-- osdep/BSDEthernetTap.cpp | 5 +- osdep/BSDEthernetTap.hpp | 4 +- osdep/LinuxEthernetTap.cpp | 4 +- osdep/LinuxEthernetTap.hpp | 4 +- osdep/OSXEthernetTap.cpp | 4 +- osdep/OSXEthernetTap.hpp | 4 +- osdep/WindowsEthernetTap.cpp | 5 +- osdep/WindowsEthernetTap.hpp | 4 +- service/OneService.cpp | 58 +++--- service/SoftwareUpdater.cpp | 12 +- 44 files changed, 673 insertions(+), 583 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 51500ed7..ce56e906 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -790,7 +790,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( test->timestamp = OSUtils::now(); if (_node) { - _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); + _node->circuitTestBegin((void *)0,test,&(EmbeddedNetworkController::_circuitTestCallback)); } else { _tests.pop_back(); return 500; diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 98413a21..747e1855 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1408,6 +1408,7 @@ typedef void ZT_Node; typedef int (*ZT_VirtualNetworkConfigFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* Network ID */ void **, /* Modifiable network user PTR */ enum ZT_VirtualNetworkConfigOperation, /* Config operation */ @@ -1423,6 +1424,7 @@ typedef int (*ZT_VirtualNetworkConfigFunction)( typedef void (*ZT_VirtualNetworkFrameFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* Network ID */ void **, /* Modifiable network user PTR */ uint64_t, /* Source MAC */ @@ -1442,10 +1444,11 @@ typedef void (*ZT_VirtualNetworkFrameFunction)( * in the definition of ZT_Event. */ typedef void (*ZT_EventCallback)( - ZT_Node *, - void *, - enum ZT_Event, - const void *); + ZT_Node *, /* Node */ + void *, /* User ptr */ + void *, /* Thread ptr */ + enum ZT_Event, /* Event type */ + const void *); /* Event payload (if applicable) */ /** * Function to get an object from the data store @@ -1468,8 +1471,9 @@ typedef void (*ZT_EventCallback)( * object. */ typedef long (*ZT_DataStoreGetFunction)( - ZT_Node *, - void *, + ZT_Node *, /* Node */ + void *, /* User ptr */ + void *, /* Thread ptr */ const char *, void *, unsigned long, @@ -1495,6 +1499,7 @@ typedef long (*ZT_DataStoreGetFunction)( typedef int (*ZT_DataStorePutFunction)( ZT_Node *, void *, + void *, /* Thread ptr */ const char *, const void *, unsigned long, @@ -1529,6 +1534,7 @@ typedef int (*ZT_DataStorePutFunction)( typedef int (*ZT_WirePacketSendFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ const struct sockaddr_storage *, /* Local address */ const struct sockaddr_storage *, /* Remote address */ const void *, /* Packet data */ @@ -1562,6 +1568,7 @@ typedef int (*ZT_WirePacketSendFunction)( typedef int (*ZT_PathCheckFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* ZeroTier address */ const struct sockaddr_storage *, /* Local address */ const struct sockaddr_storage *); /* Remote address */ @@ -1584,6 +1591,7 @@ typedef int (*ZT_PathCheckFunction)( typedef int (*ZT_PathLookupFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ + void *, /* Thread ptr */ uint64_t, /* ZeroTier address (40 bits) */ int, /* Desired ss_family or -1 for any */ struct sockaddr_storage *); /* Result buffer */ @@ -1654,11 +1662,12 @@ struct ZT_Node_Callbacks * * @param node Result: pointer is set to new node instance on success * @param uptr User pointer to pass to functions/callbacks + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param callbacks Callback function configuration * @param now Current clock in milliseconds * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); /** * Delete a node and free all resources it consumes @@ -1674,6 +1683,7 @@ void ZT_Node_delete(ZT_Node *node); * Process a packet received from the physical wire * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds * @param localAddress Local address, or point to ZT_SOCKADDR_NULL if unspecified * @param remoteAddress Origin of packet @@ -1684,6 +1694,7 @@ void ZT_Node_delete(ZT_Node *node); */ enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -1695,6 +1706,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( * Process a frame from a virtual network port (tap) * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds * @param nwid ZeroTier 64-bit virtual network ID * @param sourceMac Source MAC address (least significant 48 bits) @@ -1708,6 +1720,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( */ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -1722,11 +1735,12 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( * Perform periodic background operations * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds * @param nextBackgroundTaskDeadline Value/result: set to deadline for next call to processBackgroundTasks() * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); +enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); /** * Join a network @@ -1742,7 +1756,7 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol * @param uptr An arbitrary pointer to associate with this network (default: NULL) * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr); +enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr,void *tptr); /** * Leave a network @@ -1759,7 +1773,7 @@ enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr); * @param uptr Target pointer is set to uptr (if not NULL) * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr); +enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr,void *tptr); /** * Subscribe to an Ethernet multicast group @@ -1781,12 +1795,13 @@ enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr); * This does not generate an update call to networkConfigCallback(). * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param nwid 64-bit network ID * @param multicastGroup Ethernet multicast or broadcast MAC (least significant 48 bits) * @param multicastAdi Multicast ADI (least significant 32 bits only, use 0 if not needed) * @return OK (0) or error code if a fatal error condition has occurred */ -enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); +enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); /** * Unsubscribe from an Ethernet multicast group (or all groups) @@ -1811,21 +1826,24 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint * across invocations if the contents of moon.d are scanned and orbit is * called for each on startup. * + * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param moonWorldId Moon's world ID * @param moonSeed If non-zero, the ZeroTier address of any member of the moon to query for moon definition * @param len Length of moonWorld in bytes * @return Error if moon was invalid or failed to be added */ -enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed); +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,uint64_t moonSeed); /** * Remove a moon (does nothing if not present) * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param moonWorldId World ID of moon to remove * @return Error if anything bad happened */ -enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId); +enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId); /** * Get this node's 40-bit ZeroTier address @@ -1919,13 +1937,15 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node); * There is no delivery guarantee here. Failure can occur if the message is * too large or if dest is not a valid ZeroTier address. * + * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param dest Destination ZeroTier address * @param typeId VERB_USER_MESSAGE type ID * @param data Payload data to attach to user message * @param len Length of data in bytes * @return Boolean: non-zero on success, zero on failure */ -int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); +int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); /** * Set a network configuration master instance for this node @@ -1957,11 +1977,12 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); * for results forever. * * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param test Test configuration * @param reportCallback Function to call each time a report is received * @return OK or error if, for example, test is too big for a packet or support isn't compiled in */ -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *)); +enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *)); /** * Stop listening for results to a given circuit test diff --git a/node/Capability.cpp b/node/Capability.cpp index 0a736ca8..c178e566 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -25,7 +25,7 @@ namespace ZeroTier { -int Capability::verify(const RuntimeEnvironment *RR) const +int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const { try { // There must be at least one entry, and sanity check for bad chain max length @@ -46,12 +46,12 @@ int Capability::verify(const RuntimeEnvironment *RR) const return -1; // otherwise if we have another entry it must be from the previous holder in the chain } - const Identity id(RR->topology->getIdentity(_custody[c].from)); + const Identity id(RR->topology->getIdentity(tPtr,_custody[c].from)); if (id) { if (!id.verify(tmp.data(),tmp.size(),_custody[c].signature)) return -1; } else { - RR->sw->requestWhois(_custody[c].from); + RR->sw->requestWhois(tPtr,_custody[c].from); return 1; } } diff --git a/node/Capability.hpp b/node/Capability.hpp index d070f2ad..5ef6c994 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -161,7 +161,7 @@ public: * @param RR Runtime environment to provide for peer lookup, etc. * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template static inline void serializeRules(Buffer &b,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 43efcd20..9bf70216 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -207,14 +207,14 @@ bool CertificateOfMembership::sign(const Identity &with) } } -int CertificateOfMembership::verify(const RuntimeEnvironment *RR) const +int CertificateOfMembership::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(networkId()))||(_qualifierCount > ZT_NETWORK_COM_MAX_QUALIFIERS)) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 2d7c2cb3..ae976b50 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -250,9 +250,10 @@ public: * Verify this COM and its signature * * @param RR Runtime environment for looking up peers + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; /** * @return True if signed diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp index 6fc59ad1..2bd181e0 100644 --- a/node/CertificateOfOwnership.cpp +++ b/node/CertificateOfOwnership.cpp @@ -25,13 +25,13 @@ namespace ZeroTier { -int CertificateOfOwnership::verify(const RuntimeEnvironment *RR) const +int CertificateOfOwnership::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } try { diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 57fd8259..8c47582d 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -137,9 +137,10 @@ public: /** * @param RR Runtime environment to allow identity lookup for signedBy + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template inline void serialize(Buffer &b,const bool forSign = false) const diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e2275a04..52794fd7 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -44,7 +44,7 @@ namespace ZeroTier { -bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) +bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) { const Address sourceAddress(source()); @@ -65,10 +65,10 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { // Only HELLO is allowed in the clear, but will still have a MAC - return _doHELLO(RR,false); + return _doHELLO(RR,tPtr,false); } - const SharedPtr peer(RR->topology->getPeer(sourceAddress)); + const SharedPtr peer(RR->topology->getPeer(tPtr,sourceAddress)); if (peer) { if (!trusted) { if (!dearmor(peer->key())) { @@ -89,30 +89,30 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); return true; - case Packet::VERB_HELLO: return _doHELLO(RR,true); - case Packet::VERB_ERROR: return _doERROR(RR,peer); - case Packet::VERB_OK: return _doOK(RR,peer); - case Packet::VERB_WHOIS: return _doWHOIS(RR,peer); - case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,peer); - case Packet::VERB_FRAME: return _doFRAME(RR,peer); - case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,peer); - case Packet::VERB_ECHO: return _doECHO(RR,peer); - case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,peer); - case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,peer); - case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,peer); - case Packet::VERB_NETWORK_CONFIG: return _doNETWORK_CONFIG(RR,peer); - case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,peer); - case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,peer); - case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,peer); - case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,peer); - case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,peer); - case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,peer); + case Packet::VERB_HELLO: return _doHELLO(RR,tPtr,true); + case Packet::VERB_ERROR: return _doERROR(RR,tPtr,peer); + case Packet::VERB_OK: return _doOK(RR,tPtr,peer); + case Packet::VERB_WHOIS: return _doWHOIS(RR,tPtr,peer); + case Packet::VERB_RENDEZVOUS: return _doRENDEZVOUS(RR,tPtr,peer); + case Packet::VERB_FRAME: return _doFRAME(RR,tPtr,peer); + case Packet::VERB_EXT_FRAME: return _doEXT_FRAME(RR,tPtr,peer); + case Packet::VERB_ECHO: return _doECHO(RR,tPtr,peer); + case Packet::VERB_MULTICAST_LIKE: return _doMULTICAST_LIKE(RR,tPtr,peer); + case Packet::VERB_NETWORK_CREDENTIALS: return _doNETWORK_CREDENTIALS(RR,tPtr,peer); + case Packet::VERB_NETWORK_CONFIG_REQUEST: return _doNETWORK_CONFIG_REQUEST(RR,tPtr,peer); + case Packet::VERB_NETWORK_CONFIG: return _doNETWORK_CONFIG(RR,tPtr,peer); + case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,tPtr,peer); + case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); + case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); + case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,tPtr,peer); + case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,tPtr,peer); + case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); } } else { - RR->sw->requestWhois(sourceAddress); + RR->sw->requestWhois(tPtr,sourceAddress); return false; } } catch ( ... ) { @@ -123,7 +123,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR) } } -bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB]; @@ -163,7 +163,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr case Packet::ERROR_IDENTITY_COLLISION: // FIXME: for federation this will need a payload with a signature or something. if (RR->topology->isUpstream(peer->identity())) - RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); + RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); break; case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { @@ -171,7 +171,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); const uint64_t now = RR->node->now(); if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) - network->pushCredentialsNow(peer->address(),now); + network->pushCredentialsNow(tPtr,peer->address(),now); } break; case Packet::ERROR_NETWORK_ACCESS_DENIED_: { @@ -185,7 +185,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr // Members of networks can use this error to indicate that they no longer // want to receive multicasts on a given channel. const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->gate(peer))) { + if ((network)&&(network->gate(tPtr,peer))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); RR->mc->remove(network->id(),mg,peer->address()); @@ -195,14 +195,14 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated) +bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated) { try { const uint64_t now = RR->node->now(); @@ -226,7 +226,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } - SharedPtr peer(RR->topology->getPeer(id.address())); + SharedPtr peer(RR->topology->getPeer(tPtr,id.address())); if (peer) { // We already have an identity with this address -- check for collisions if (!alreadyAuthenticated) { @@ -246,7 +246,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut outp.append((uint64_t)pid); outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION); outp.armor(key,true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } else { TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); } @@ -292,7 +292,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut return true; } - peer = RR->topology->addPeer(newPeer); + peer = RR->topology->addPeer(tPtr,newPeer); // Continue at // VALID } @@ -304,7 +304,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut if (ptr < size()) { ptr += externalSurfaceAddress.deserialize(*this,ptr); if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + RR->sa->iam(tPtr,id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); } // Get primary planet world ID and world timestamp if present @@ -408,17 +408,17 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),now); + _path->send(RR,tPtr,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; @@ -463,7 +463,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p while (ptr < endOfWorlds) { World w; ptr += w.deserialize(*this,ptr); - RR->topology->addWorld(w,false); + RR->topology->addWorld(tPtr,w,false); } } else { ptr += worldsLen; @@ -490,20 +490,20 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); + RR->sa->iam(tPtr,peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; case Packet::VERB_WHOIS: if (RR->topology->isUpstream(peer->identity())) { const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY); - RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr(new Peer(RR,RR->identity,id)))); + RR->sw->doAnythingWaitingForPeer(tPtr,RR->topology->addPeer(tPtr,SharedPtr(new Peer(RR,RR->identity,id)))); } break; case Packet::VERB_NETWORK_CONFIG_REQUEST: { const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); if (network) - network->handleConfigChunk(packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); + network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); } break; case Packet::VERB_MULTICAST_GATHER: { @@ -513,7 +513,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); } } break; @@ -532,7 +532,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); if (com) - network->addCredential(com); + network->addCredential(tPtr,com); } if ((flags & 0x02) != 0) { @@ -540,7 +540,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; unsigned int totalKnown = at(offset); offset += 4; unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); } } } break; @@ -548,14 +548,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr &p default: break; } - peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) { @@ -573,13 +573,13 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr const Address addr(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const Identity id(RR->topology->getIdentity(addr)); + const Identity id(RR->topology->getIdentity(tPtr,addr)); if (id) { id.serialize(outp,false); ++count; } else { // Request unknown WHOIS from upstream from us (if we have one) - RR->sw->requestWhois(addr); + RR->sw->requestWhois(tPtr,addr); #ifdef ZT_ENABLE_CLUSTER // Distribute WHOIS queries across a cluster if we do not know the ID. // This may result in duplicate OKs to the querying peer, which is fine. @@ -591,32 +591,32 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr if (count > 0) { outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (!RR->topology->isUpstream(peer->identity())) { TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str()); } else { const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr rendezvousWith(RR->topology->getPeer(with)); + const SharedPtr rendezvousWith(RR->topology->getPeer(tPtr,with)); if (rendezvousWith) { const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (RR->node->shouldUsePathForZeroTierTraffic(with,_path->localAddress(),atAddr)) { - RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now(),false,0); + if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localAddress(),atAddr)) { + RR->node->putPacket(tPtr,_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->attemptToContactAt(tPtr,_path->localAddress(),atAddr,RR->node->now(),false,0); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -628,46 +628,46 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr< TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); const SharedPtr network(RR->node->network(nwid)); bool trustEstablished = false; if (network) { - if (network->gate(peer)) { + if (network->gate(tPtr,peer)) { trustEstablished = true; if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); const MAC sourceMac(peer->address(),nwid); const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) - RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) + RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); } } else { TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } } else { TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } - peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); @@ -680,13 +680,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

addCredential(com); + network->addCredential(tPtr,com); } - if (!network->gate(peer)) { + if (!network->gate(tPtr,peer)) { TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -699,36 +699,36 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

mac())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } - switch (network->filterIncomingPacket(peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { + switch (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { case 1: if (from != MAC(peer->address(),nwid)) { if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (to.isMulticast()) { if (network->config().multicastLimit == 0) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: network %.16llx does not allow multicast",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (!network->config().permitsBridging(RR->identity.address())) { TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } // fall through -- 2 means accept regardless of bridging checks or other restrictions case 2: - RR->node->putFrame(nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); + RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); break; } } @@ -739,14 +739,14 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } else { TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -754,7 +754,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr

&peer) +bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (!peer->rateGateEchoRequest(RR->node->now())) { @@ -769,16 +769,16 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr if (size() > ZT_PACKET_IDX_PAYLOAD) outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t now = RR->node->now(); @@ -802,9 +802,9 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared if (!auth) { if ((!network)||(network->id() != nwid)) network = RR->node->network(nwid); - const bool authOnNet = ((network)&&(network->gate(peer))); + const bool authOnNet = ((network)&&(network->gate(tPtr,peer))); if (!authOnNet) - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); trustEstablished |= authOnNet; if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { auth = true; @@ -815,18 +815,18 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared if (auth) { const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); - RR->mc->add(now,nwid,group,peer->address()); + RR->mc->add(tPtr,now,nwid,group,peer->address()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (!peer->rateGateCredentialsReceived(RR->node->now())) { @@ -847,7 +847,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S if (com) { const SharedPtr network(RR->node->network(com.networkId())); if (network) { - switch (network->addCredential(com)) { + switch (network->addCredential(tPtr,com)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -857,7 +857,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S case Membership::ADD_DEFERRED_FOR_WHOIS: return false; } - } else RR->mc->addCredential(com,false); + } else RR->mc->addCredential(tPtr,com,false); } } ++p; // skip trailing 0 after COMs if present @@ -868,7 +868,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += cap.deserialize(*this,p); const SharedPtr network(RR->node->network(cap.networkId())); if (network) { - switch (network->addCredential(cap)) { + switch (network->addCredential(tPtr,cap)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -888,7 +888,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += tag.deserialize(*this,p); const SharedPtr network(RR->node->network(tag.networkId())); if (network) { - switch (network->addCredential(tag)) { + switch (network->addCredential(tPtr,tag)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -908,7 +908,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += revocation.deserialize(*this,p); const SharedPtr network(RR->node->network(revocation.networkId())); if (network) { - switch(network->addCredential(peer->address(),revocation)) { + switch(network->addCredential(tPtr,peer->address(),revocation)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -928,7 +928,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S p += coo.deserialize(*this,p); const SharedPtr network(RR->node->network(coo.networkId())); if (network) { - switch(network->addCredential(coo)) { + switch(network->addCredential(tPtr,coo)) { case Membership::ADD_REJECTED: break; case Membership::ADD_ACCEPTED_NEW: @@ -942,7 +942,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); } catch (std::exception &exc) { //fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -953,7 +953,7 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S return true; } -bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); @@ -972,10 +972,10 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); outp.append(nwid); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); } catch (std::exception &exc) { //fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); @@ -986,12 +986,12 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons return true; } -bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); if (network) { - const uint64_t configUpdateId = network->handleConfigChunk(packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); + const uint64_t configUpdateId = network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); if (configUpdateId) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((uint8_t)Packet::VERB_ECHO); @@ -999,17 +999,17 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,const Shared outp.append((uint64_t)network->id()); outp.append((uint64_t)configUpdateId); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); @@ -1027,17 +1027,17 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); if (com) { if (network) - network->addCredential(com); - else RR->mc->addCredential(com,false); + network->addCredential(tPtr,com); + else RR->mc->addCredential(tPtr,com,false); } } catch ( ... ) { TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str()); } } - const bool trustEstablished = ((network)&&(network->gate(peer))); + const bool trustEstablished = ((network)&&(network->gate(tPtr,peer))); if (!trustEstablished) - _sendErrorNeedCredentials(RR,peer,nwid); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); @@ -1048,7 +1048,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); if (gatheredLocally > 0) { outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } // If we are a member of a cluster, distribute this GATHER across it @@ -1058,14 +1058,14 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar #endif } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID); @@ -1081,19 +1081,19 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share CertificateOfMembership com; offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); if (com) - network->addCredential(com); + network->addCredential(tPtr,com); } - if (!network->gate(peer)) { + if (!network->gate(tPtr,peer)) { TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } if (network->config().multicastLimit == 0) { TRACE("dropped MULTICAST_FRAME from %s(%s): network %.16llx does not allow multicast",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } @@ -1120,12 +1120,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -1134,14 +1134,14 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share network->learnBridgeRoute(from,peer->address()); } else { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (network->filterIncomingPacket(peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { - RR->node->putFrame(nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { + RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); } } @@ -1155,14 +1155,14 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share outp.append((unsigned char)0x02); // flag 0x02 = contains gather results if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) { outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),RR->node->now()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } } - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); } else { - _sendErrorNeedCredentials(RR,peer,nwid); - peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); @@ -1170,7 +1170,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share return true; } -bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const uint64_t now = RR->node->now(); @@ -1178,7 +1178,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1209,10 +1209,10 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now,false,0); + peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1228,10 +1228,10 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha redundant = peer->hasActivePathTo(now,a); } - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) { + if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); - peer->attemptToContactAt(InetAddress(),a,now,false,0); + peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } @@ -1241,20 +1241,20 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha ptr += addrLen; } - peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { const Address originatorAddress(field(ZT_PACKET_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - SharedPtr originator(RR->topology->getPeer(originatorAddress)); + SharedPtr originator(RR->topology->getPeer(tPtr,originatorAddress)); if (!originator) { - RR->sw->requestWhois(originatorAddress); + RR->sw->requestWhois(tPtr,originatorAddress); return false; } @@ -1285,7 +1285,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } vlf += signatureLength; @@ -1304,14 +1304,14 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt SharedPtr network(RR->node->network(originatorCredentialNetworkId)); if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } - if (network->gate(peer)) + if (network->gate(tPtr,peer)) reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; } else { TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); return true; } @@ -1327,7 +1327,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt for(unsigned int h=0;h nhp(RR->topology->getPeer(nextHop[h])); + SharedPtr nhp(RR->topology->getPeer(tPtr,nextHop[h])); if (nhp) { SharedPtr nhbp(nhp->getBestPath(now,false)); if ((nhbp)&&(nhbp->alive(now))) @@ -1362,7 +1362,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt nextHop[h].appendTo(outp); nextHopBestPathAddress[h].serialize(outp); // appends 0 if null InetAddress } - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } // If there are next hops, forward the test along through the graph @@ -1377,19 +1377,19 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt if (RR->identity.address() != nextHop[h]) { // next hops that loop back to the current hop are not valid outp.newInitializationVector(); outp.setDestination(nextHop[h]); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } } - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { ZT_CircuitTestReport report; @@ -1431,14 +1431,14 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S RR->node->postCircuitTestReport(&report); - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPtr &peer) +bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { if (size() >= (ZT_PACKET_IDX_PAYLOAD + 8)) { @@ -1447,16 +1447,16 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPt um.typeId = at(ZT_PACKET_IDX_PAYLOAD); um.data = reinterpret_cast(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD + 8); um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); - RR->node->postEvent(ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); + RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); } - peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); } catch ( ... ) { TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } -void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid) +void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid) { const uint64_t now = RR->node->now(); if (peer->rateGateOutgoingComRequest(now)) { @@ -1466,7 +1466,7 @@ void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,cons outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE); outp.append(nwid); outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,outp.data(),outp.size(),now); + _path->send(RR,tPtr,outp.data(),outp.size(),now); } } diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index febff28a..3d4a2e05 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -102,9 +102,10 @@ public: * may no longer be valid. * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return True if decoding and processing is complete, false if caller should try again */ - bool tryDecode(const RuntimeEnvironment *RR); + bool tryDecode(const RuntimeEnvironment *RR,void *tPtr); /** * @return Time of packet receipt / start of decode @@ -114,26 +115,26 @@ public: private: // These are called internally to handle packet contents once it has // been authenticated, decrypted, decompressed, and classified. - bool _doERROR(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated); - bool _doOK(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doWHOIS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doFRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doECHO(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doNETWORK_CONFIG(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr &peer); - bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,const SharedPtr &peer); - - void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,const SharedPtr &peer,const uint64_t nwid); + bool _doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated); + bool _doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doFRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + + void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid); uint64_t _receiveTime; SharedPtr _path; diff --git a/node/Membership.cpp b/node/Membership.cpp index 3b2e3b1c..22c13c88 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -40,7 +40,7 @@ Membership::Membership() : for(unsigned int i=0;i= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); @@ -113,7 +113,7 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,const uint64_t now outp.setAt(cooCountAt,(uint16_t)thisPacketCooCount); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } @@ -123,7 +123,7 @@ const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) cons return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) { const uint64_t newts = com.timestamp().first; if (newts <= _comRevocationThreshold) { @@ -141,7 +141,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme return ADD_ACCEPTED_REDUNDANT; } - switch(com.verify(RR)) { + switch(com.verify(RR,tPtr)) { default: TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (invalid signature or object)",com.issuedTo().toString().c_str(),com.networkId()); return ADD_REJECTED; @@ -154,7 +154,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; @@ -169,7 +169,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - switch(tag.verify(RR)) { + switch(tag.verify(RR,tPtr)) { default: TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); return ADD_REJECTED; @@ -184,7 +184,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; @@ -199,7 +199,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - switch(cap.verify(RR)) { + switch(cap.verify(RR,tPtr)) { default: TRACE("addCredential(Capability) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); return ADD_REJECTED; @@ -214,9 +214,9 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { - switch(rev.verify(RR)) { + switch(rev.verify(RR,tPtr)) { default: return ADD_REJECTED; case 0: { @@ -239,7 +239,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo) +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; @@ -254,7 +254,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } - switch(coo.verify(RR)) { + switch(coo.verify(RR,tPtr)) { default: TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",coo.issuedTo().toString().c_str(),coo.networkId()); return ADD_REJECTED; diff --git a/node/Membership.hpp b/node/Membership.hpp index 97510b57..c28d598c 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -158,13 +158,14 @@ public: * sends VERB_NETWORK_CREDENTIALS if the recipient might need them. * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param peerAddress Address of member peer (the one that this Membership describes) * @param nconf My network config * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none * @param force If true, send objects regardless of last push time */ - void pushCredentials(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); + void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** * Check whether we should push MULTICAST_LIKEs to this peer @@ -226,27 +227,27 @@ public: /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfMembership &com); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Tag &tag); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Capability &cap); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const Revocation &rev); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,const NetworkConfig &nconf,const CertificateOfOwnership &coo); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo); private: _RemoteCredential *_newTag(const uint64_t id); diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index f8d58501..8e534b5e 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -43,14 +43,14 @@ Multicaster::~Multicaster() { } -void Multicaster::addMultiple(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) +void Multicaster::addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) { const unsigned char *p = (const unsigned char *)addresses; const unsigned char *e = p + (5 * count); Mutex::Lock _l(_groups_m); MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)]; while (p != e) { - _add(now,nwid,mg,gs,Address(p,5)); + _add(tPtr,now,nwid,mg,gs,Address(p,5)); p += 5; } } @@ -152,6 +152,7 @@ std::vector

Multicaster::getMembers(uint64_t nwid,const MulticastGroup } void Multicaster::send( + void *tPtr, unsigned int limit, uint64_t now, uint64_t nwid, @@ -207,7 +208,7 @@ void Multicaster::send( for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { if (*ast != RR->identity.address()) { - out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send + out.sendOnly(RR,tPtr,*ast); // optimization: don't use dedup log if it's a one-pass send if (++count >= limit) break; } @@ -217,7 +218,7 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { Address ma(gs.members[indexes[idx++]].address); if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { - out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send + out.sendOnly(RR,tPtr,ma); // optimization: don't use dedup log if it's a one-pass send ++count; } } @@ -256,7 +257,7 @@ void Multicaster::send( if (com) com->serialize(outp); RR->node->expectReplyTo(outp.packetId()); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } @@ -280,7 +281,7 @@ void Multicaster::send( for(std::vector
::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) { if (*ast != RR->identity.address()) { - out.sendAndLog(RR,*ast); + out.sendAndLog(RR,tPtr,*ast); if (++count >= limit) break; } @@ -290,7 +291,7 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { Address ma(gs.members[indexes[idx++]].address); if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) { - out.sendAndLog(RR,ma); + out.sendAndLog(RR,tPtr,ma); ++count; } } @@ -352,15 +353,15 @@ void Multicaster::clean(uint64_t now) } } -void Multicaster::addCredential(const CertificateOfMembership &com,bool alreadyValidated) +void Multicaster::addCredential(void *tPtr,const CertificateOfMembership &com,bool alreadyValidated) { - if ((alreadyValidated)||(com.verify(RR) == 0)) { + if ((alreadyValidated)||(com.verify(RR,tPtr) == 0)) { Mutex::Lock _l(_gatherAuth_m); _gatherAuth[_GatherAuthKey(com.networkId(),com.issuedTo())] = RR->node->now(); } } -void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) +void Multicaster::_add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) { // assumes _groups_m is locked @@ -383,7 +384,7 @@ void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,Multi if (tx->atLimit()) gs.txQueue.erase(tx++); else { - tx->sendIfNew(RR,member); + tx->sendIfNew(RR,tPtr,member); if (tx->atLimit()) gs.txQueue.erase(tx++); else ++tx; diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 32dec9cf..f646a5be 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -90,10 +90,10 @@ public: * @param mg Multicast group * @param member New member address */ - inline void add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) + inline void add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) { Mutex::Lock _l(_groups_m); - _add(now,nwid,mg,_groups[Multicaster::Key(nwid,mg)],member); + _add(tPtr,now,nwid,mg,_groups[Multicaster::Key(nwid,mg)],member); } /** @@ -101,6 +101,7 @@ public: * * It's up to the caller to check bounds on the array before calling this. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param nwid Network ID * @param mg Multicast group @@ -108,7 +109,7 @@ public: * @param count Number of addresses * @param totalKnown Total number of known addresses as reported by peer */ - void addMultiple(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); + void addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); /** * Remove a multicast group member (if present) @@ -150,6 +151,7 @@ public: /** * Send a multicast * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param limit Multicast limit * @param now Current time * @param nwid Network ID @@ -162,6 +164,7 @@ public: * @param len Length of packet data */ void send( + void *tPtr, unsigned int limit, uint64_t now, uint64_t nwid, @@ -191,7 +194,7 @@ public: * @param com Certificate of membership * @param alreadyValidated If true, COM has already been checked and found to be valid and signed */ - void addCredential(const CertificateOfMembership &com,bool alreadyValidated); + void addCredential(void *tPtr,const CertificateOfMembership &com,bool alreadyValidated); /** * Check authorization for GATHER and LIKE for non-network-members @@ -209,7 +212,7 @@ public: } private: - void _add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); + void _add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); const RuntimeEnvironment *RR; diff --git a/node/Network.cpp b/node/Network.cpp index 38c1b0d9..0abfdf86 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -674,7 +674,7 @@ static _doZtFilterResult _doZtFilter( const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); -Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : +Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr) : RR(renv), _uPtr(uptr), _id(nwid), @@ -696,11 +696,11 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : Dictionary *dconf = new Dictionary(); NetworkConfig *nconf = new NetworkConfig(); try { - std::string conf(RR->node->dataStoreGet(confn)); + std::string conf(RR->node->dataStoreGet(tPtr,confn)); if (conf.length()) { dconf->load(conf.c_str()); if (nconf->fromDictionary(*dconf)) { - this->setConfiguration(*nconf,false); + this->setConfiguration(tPtr,*nconf,false); _lastConfigUpdate = 0; // we still want to re-request a new config from the network gotConf = true; } @@ -711,13 +711,13 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr) : if (!gotConf) { // Save a one-byte CR to persist membership while we request a real netconf - RR->node->dataStorePut(confn,"\n",1,false); + RR->node->dataStorePut(tPtr,confn,"\n",1,false); } if (!_portInitialized) { ZT_VirtualNetworkConfig ctmp; _externalConfig(&ctmp); - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); _portInitialized = true; } } @@ -729,15 +729,16 @@ Network::~Network() char n[128]; if (_destroyed) { - RR->node->configureVirtualNetworkPort(_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - RR->node->dataStoreDelete(n); + RR->node->dataStoreDelete((void *)0,n); } else { - RR->node->configureVirtualNetworkPort(_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp); + RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp); } } bool Network::filterOutgoingPacket( + void *tPtr, const bool noTee, const Address &ztSource, const Address &ztDest, @@ -781,7 +782,7 @@ bool Network::filterOutgoingPacket( if ((!noTee)&&(cc2)) { Membership &m2 = _membership(cc2); - m2.pushCredentials(RR,now,cc2,_config,localCapabilityIndex,false); + m2.pushCredentials(RR,tPtr,now,cc2,_config,localCapabilityIndex,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -791,7 +792,7 @@ bool Network::filterOutgoingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength2); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } break; @@ -813,11 +814,11 @@ bool Network::filterOutgoingPacket( if (accept) { if (membership) - membership->pushCredentials(RR,now,ztDest,_config,localCapabilityIndex,false); + membership->pushCredentials(RR,tPtr,now,ztDest,_config,localCapabilityIndex,false); if ((!noTee)&&(cc)) { Membership &m2 = _membership(cc); - m2.pushCredentials(RR,now,cc,_config,localCapabilityIndex,false); + m2.pushCredentials(RR,tPtr,now,cc,_config,localCapabilityIndex,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -827,12 +828,12 @@ bool Network::filterOutgoingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { Membership &m2 = _membership(ztFinalDest); - m2.pushCredentials(RR,now,ztFinalDest,_config,localCapabilityIndex,false); + m2.pushCredentials(RR,tPtr,now,ztFinalDest,_config,localCapabilityIndex,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -842,7 +843,7 @@ bool Network::filterOutgoingPacket( outp.append((uint16_t)etherType); outp.append(frameData,frameLen); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); return false; // DROP locally, since we redirected } else { @@ -854,6 +855,7 @@ bool Network::filterOutgoingPacket( } int Network::filterIncomingPacket( + void *tPtr, const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, @@ -898,7 +900,7 @@ int Network::filterIncomingPacket( if (accept) { if (cc2) { - _membership(cc2).pushCredentials(RR,RR->node->now(),cc2,_config,-1,false); + _membership(cc2).pushCredentials(RR,tPtr,RR->node->now(),cc2,_config,-1,false); Packet outp(cc2,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -908,7 +910,7 @@ int Network::filterIncomingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength2); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } break; } @@ -929,7 +931,7 @@ int Network::filterIncomingPacket( if (accept) { if (cc) { - _membership(cc).pushCredentials(RR,RR->node->now(),cc,_config,-1,false); + _membership(cc).pushCredentials(RR,tPtr,RR->node->now(),cc,_config,-1,false); Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -939,11 +941,11 @@ int Network::filterIncomingPacket( outp.append((uint16_t)etherType); outp.append(frameData,ccLength); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } if ((ztDest != ztFinalDest)&&(ztFinalDest)) { - _membership(ztFinalDest).pushCredentials(RR,RR->node->now(),ztFinalDest,_config,-1,false); + _membership(ztFinalDest).pushCredentials(RR,tPtr,RR->node->now(),ztFinalDest,_config,-1,false); Packet outp(ztFinalDest,RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(_id); @@ -953,7 +955,7 @@ int Network::filterIncomingPacket( outp.append((uint16_t)etherType); outp.append(frameData,frameLen); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); return 0; // DROP locally, since we redirected } @@ -972,12 +974,12 @@ bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBr return false; } -void Network::multicastSubscribe(const MulticastGroup &mg) +void Network::multicastSubscribe(void *tPtr,const MulticastGroup &mg) { Mutex::Lock _l(_lock); if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) { _myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg); - _sendUpdatesToMembers(&mg); + _sendUpdatesToMembers(tPtr,&mg); } } @@ -989,7 +991,7 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) _myMulticastGroups.erase(i); } -uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) +uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) { const unsigned int start = ptr; @@ -1043,7 +1045,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc } // If it's not a duplicate, check chunk signature - const Identity controllerId(RR->topology->getIdentity(controller())); + const Identity controllerId(RR->topology->getIdentity(tPtr,controller())); if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? TRACE("unable to verify chunk from %s: don't have controller identity",source.toString().c_str()); return 0; @@ -1067,7 +1069,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc if ((*a != source)&&(*a != controller())) { Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CONFIG); outp.append(reinterpret_cast(chunk.data()) + start,chunk.size() - start); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } } @@ -1126,7 +1128,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc } if (nc) { - this->setConfiguration(*nc,true); + this->setConfiguration(tPtr,*nc,true); delete nc; return configUpdateId; } else { @@ -1136,7 +1138,7 @@ uint64_t Network::handleConfigChunk(const uint64_t packetId,const Address &sourc return 0; } -int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) +int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToDisk) { // _lock is NOT locked when this is called try { @@ -1156,7 +1158,7 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) _portInitialized = true; _externalConfig(&ctmp); } - _portError = RR->node->configureVirtualNetworkPort(_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { Dictionary *d = new Dictionary(); @@ -1164,7 +1166,7 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) char n[64]; Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); if (nconf.toDictionary(*d,false)) - RR->node->dataStorePut(n,(const void *)d->data(),d->sizeBytes(),true); + RR->node->dataStorePut(tPtr,n,(const void *)d->data(),d->sizeBytes(),true); } catch ( ... ) {} delete d; } @@ -1176,7 +1178,7 @@ int Network::setConfiguration(const NetworkConfig &nconf,bool saveToDisk) return 0; } -void Network::requestConfiguration() +void Network::requestConfiguration(void *tPtr) { /* ZeroTier addresses can't begin with 0xff, so this is used to mark controllerless * network IDs. Controllerless network IDs only support unicast IPv6 using the 6plane @@ -1236,7 +1238,7 @@ void Network::requestConfiguration() nconf->type = ZT_NETWORK_TYPE_PUBLIC; Utils::snprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); - this->setConfiguration(*nconf,false); + this->setConfiguration(tPtr,*nconf,false); delete nconf; } else { this->setNotFound(); @@ -1284,10 +1286,10 @@ void Network::requestConfiguration() } outp.compress(); RR->node->expectReplyTo(outp.packetId()); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } -bool Network::gate(const SharedPtr &peer) +bool Network::gate(void *tPtr,const SharedPtr &peer) { const uint64_t now = RR->node->now(); Mutex::Lock _l(_lock); @@ -1298,8 +1300,8 @@ bool Network::gate(const SharedPtr &peer) if (!m) m = &(_membership(peer->address())); if (m->shouldLikeMulticasts(now)) { - m->pushCredentials(RR,now,peer->address(),_config,-1,false); - _announceMulticastGroupsTo(peer->address(),_allMulticastGroups()); + m->pushCredentials(RR,tPtr,now,peer->address(),_config,-1,false); + _announceMulticastGroupsTo(tPtr,peer->address(),_allMulticastGroups()); m->likingMulticasts(now); } return true; @@ -1377,31 +1379,31 @@ void Network::learnBridgeRoute(const MAC &mac,const Address &addr) } } -void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now) +void Network::learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now) { Mutex::Lock _l(_lock); const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); _multicastGroupsBehindMe.set(mg,now); if (tmp != _multicastGroupsBehindMe.size()) - _sendUpdatesToMembers(&mg); + _sendUpdatesToMembers(tPtr,&mg); } -Membership::AddCredentialResult Network::addCredential(const CertificateOfMembership &com) +Membership::AddCredentialResult Network::addCredential(void *tPtr,const CertificateOfMembership &com) { if (com.networkId() != _id) return Membership::ADD_REJECTED; const Address a(com.issuedTo()); Mutex::Lock _l(_lock); Membership &m = _membership(a); - const Membership::AddCredentialResult result = m.addCredential(RR,_config,com); + const Membership::AddCredentialResult result = m.addCredential(RR,tPtr,_config,com); if ((result == Membership::ADD_ACCEPTED_NEW)||(result == Membership::ADD_ACCEPTED_REDUNDANT)) { - m.pushCredentials(RR,RR->node->now(),a,_config,-1,false); - RR->mc->addCredential(com,true); + m.pushCredentials(RR,tPtr,RR->node->now(),a,_config,-1,false); + RR->mc->addCredential(tPtr,com,true); } return result; } -Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,const Revocation &rev) +Membership::AddCredentialResult Network::addCredential(void *tPtr,const Address &sentFrom,const Revocation &rev) { if (rev.networkId() != _id) return Membership::ADD_REJECTED; @@ -1409,7 +1411,7 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c Mutex::Lock _l(_lock); Membership &m = _membership(rev.target()); - const Membership::AddCredentialResult result = m.addCredential(RR,_config,rev); + const Membership::AddCredentialResult result = m.addCredential(RR,tPtr,_config,rev); if ((result == Membership::ADD_ACCEPTED_NEW)&&(rev.fastPropagate())) { Address *a = (Address *)0; @@ -1424,7 +1426,7 @@ Membership::AddCredentialResult Network::addCredential(const Address &sentFrom,c outp.append((uint16_t)1); // one revocation! rev.serialize(outp); outp.append((uint16_t)0); // no certificates of ownership - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } } @@ -1495,7 +1497,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const } } -void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup) +void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked const uint64_t now = RR->node->now(); @@ -1521,9 +1523,9 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou outp.append((uint16_t)0); // no tags outp.append((uint16_t)0); // no revocations outp.append((uint16_t)0); // no certificates of ownership - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } - _announceMulticastGroupsTo(*a,groups); + _announceMulticastGroupsTo(tPtr,*a,groups); } // Also announce to controller, and send COM to simplify and generalize behavior even though in theory it does not need it @@ -1537,9 +1539,9 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou outp.append((uint16_t)0); // no tags outp.append((uint16_t)0); // no revocations outp.append((uint16_t)0); // no certificates of ownership - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } - _announceMulticastGroupsTo(c,groups); + _announceMulticastGroupsTo(tPtr,c,groups); } } @@ -1556,17 +1558,17 @@ void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGrou Membership *m = (Membership *)0; Hashtable::Iterator i(_memberships); while (i.next(a,m)) { - m->pushCredentials(RR,now,*a,_config,-1,false); + m->pushCredentials(RR,tPtr,now,*a,_config,-1,false); if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { if (!newMulticastGroup) m->likingMulticasts(now); - _announceMulticastGroupsTo(*a,groups); + _announceMulticastGroupsTo(tPtr,*a,groups); } } } } -void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups) +void Network::_announceMulticastGroupsTo(void *tPtr,const Address &peer,const std::vector &allMulticastGroups) { // Assumes _lock is locked Packet outp(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); @@ -1574,7 +1576,7 @@ void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) { if ((outp.size() + 24) >= ZT_PROTO_MAX_PACKET_LENGTH) { outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); outp.reset(peer,RR->identity.address(),Packet::VERB_MULTICAST_LIKE); } @@ -1586,7 +1588,7 @@ void Network::_announceMulticastGroupsTo(const Address &peer,const std::vector ZT_PROTO_MIN_PACKET_LENGTH) { outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tPtr,outp,true); } } diff --git a/node/Network.hpp b/node/Network.hpp index 6cf6d974..fccc267a 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -77,10 +77,11 @@ public: * constructed to actually configure the port. * * @param renv Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param nwid Network ID * @param uptr Arbitrary pointer used by externally-facing API (for user use) */ - Network(const RuntimeEnvironment *renv,uint64_t nwid,void *uptr); + Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr); ~Network(); @@ -101,6 +102,7 @@ public: * such as TEE may be taken, and credentials may be pushed, so this is not * side-effect-free. It's basically step one in sending something over VL2. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param noTee If true, do not TEE anything anywhere (for two-pass filtering as done with multicast and bridging) * @param ztSource Source ZeroTier address * @param ztDest Destination ZeroTier address @@ -113,6 +115,7 @@ public: * @return True if packet should be sent, false if dropped or redirected */ bool filterOutgoingPacket( + void *tPtr, const bool noTee, const Address &ztSource, const Address &ztDest, @@ -131,6 +134,7 @@ public: * a match certain actions may be taken such as sending a copy of the packet * to a TEE or REDIRECT target. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param sourcePeer Source Peer * @param ztDest Destination ZeroTier address * @param macSource Ethernet layer source address @@ -142,6 +146,7 @@ public: * @return 0 == drop, 1 == accept, 2 == accept even if bridged */ int filterIncomingPacket( + void *tPtr, const SharedPtr &sourcePeer, const Address &ztDest, const MAC &macSource, @@ -163,9 +168,10 @@ public: /** * Subscribe to a multicast group * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param mg New multicast group */ - void multicastSubscribe(const MulticastGroup &mg); + void multicastSubscribe(void *tPtr,const MulticastGroup &mg); /** * Unsubscribe from a multicast group @@ -181,22 +187,24 @@ public: * chunks via OK(NETWORK_CONFIG_REQUEST) or NETWORK_CONFIG. It verifies * each chunk and once assembled applies the configuration. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param packetId Packet ID or 0 if none (e.g. via cluster path) * @param source Address of sender of chunk or NULL if none (e.g. via cluster path) * @param chunk Buffer containing chunk * @param ptr Index of chunk and related fields in packet * @return Update ID if update was fully assembled and accepted or 0 otherwise */ - uint64_t handleConfigChunk(const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr); + uint64_t handleConfigChunk(void *tPtr,const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr); /** * Set network configuration * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param nconf Network configuration * @param saveToDisk Save to disk? Used during loading, should usually be true otherwise. * @return 0 == bad, 1 == accepted but duplicate/unchanged, 2 == accepted and new */ - int setConfiguration(const NetworkConfig &nconf,bool saveToDisk); + int setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToDisk); /** * Set netconf failure to 'access denied' -- called in IncomingPacket when controller reports this @@ -218,13 +226,18 @@ public: /** * Causes this network to request an updated configuration from its master node now + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call */ - void requestConfiguration(); + void requestConfiguration(void *tPtr); /** * Determine whether this peer is permitted to communicate on this network + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param peer Peer to check */ - bool gate(const SharedPtr &peer); + bool gate(void *tPtr,const SharedPtr &peer); /** * Do periodic cleanup and housekeeping tasks @@ -233,11 +246,13 @@ public: /** * Push state to members such as multicast group memberships and latest COM (if needed) + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call */ - inline void sendUpdatesToMembers() + inline void sendUpdatesToMembers(void *tPtr) { Mutex::Lock _l(_lock); - _sendUpdatesToMembers((const MulticastGroup *)0); + _sendUpdatesToMembers(tPtr,(const MulticastGroup *)0); } /** @@ -264,64 +279,66 @@ public: /** * Learn a multicast group that is bridged to our tap device * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param mg Multicast group * @param now Current time */ - void learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now); + void learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now); /** * Validate a credential and learn it if it passes certificate and other checks */ - Membership::AddCredentialResult addCredential(const CertificateOfMembership &com); + Membership::AddCredentialResult addCredential(void *tPtr,const CertificateOfMembership &com); /** * Validate a credential and learn it if it passes certificate and other checks */ - inline Membership::AddCredentialResult addCredential(const Capability &cap) + inline Membership::AddCredentialResult addCredential(void *tPtr,const Capability &cap) { if (cap.networkId() != _id) return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(cap.issuedTo()).addCredential(RR,_config,cap); + return _membership(cap.issuedTo()).addCredential(RR,tPtr,_config,cap); } /** * Validate a credential and learn it if it passes certificate and other checks */ - inline Membership::AddCredentialResult addCredential(const Tag &tag) + inline Membership::AddCredentialResult addCredential(void *tPtr,const Tag &tag) { if (tag.networkId() != _id) return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(tag.issuedTo()).addCredential(RR,_config,tag); + return _membership(tag.issuedTo()).addCredential(RR,tPtr,_config,tag); } /** * Validate a credential and learn it if it passes certificate and other checks */ - Membership::AddCredentialResult addCredential(const Address &sentFrom,const Revocation &rev); + Membership::AddCredentialResult addCredential(void *tPtr,const Address &sentFrom,const Revocation &rev); /** * Validate a credential and learn it if it passes certificate and other checks */ - inline Membership::AddCredentialResult addCredential(const CertificateOfOwnership &coo) + inline Membership::AddCredentialResult addCredential(void *tPtr,const CertificateOfOwnership &coo) { if (coo.networkId() != _id) return Membership::ADD_REJECTED; Mutex::Lock _l(_lock); - return _membership(coo.issuedTo()).addCredential(RR,_config,coo); + return _membership(coo.issuedTo()).addCredential(RR,tPtr,_config,coo); } /** * Force push credentials (COM, etc.) to a peer now * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param to Destination peer address * @param now Current time */ - inline void pushCredentialsNow(const Address &to,const uint64_t now) + inline void pushCredentialsNow(void *tPtr,const Address &to,const uint64_t now) { Mutex::Lock _l(_lock); - _membership(to).pushCredentials(RR,now,to,_config,-1,true); + _membership(to).pushCredentials(RR,tPtr,now,to,_config,-1,true); } /** @@ -353,8 +370,8 @@ private: ZT_VirtualNetworkStatus _status() const; void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked bool _gate(const SharedPtr &peer); - void _sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup); - void _announceMulticastGroupsTo(const Address &peer,const std::vector &allMulticastGroups); + void _sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup); + void _announceMulticastGroupsTo(void *tPtr,const Address &peer,const std::vector &allMulticastGroups); std::vector _allMulticastGroups() const; Membership &_membership(const Address &a); diff --git a/node/Node.cpp b/node/Node.cpp index 1125ca7a..4e8d6655 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -46,7 +46,7 @@ namespace ZeroTier { /* Public Node interface (C++, exposed via CAPI bindings) */ /****************************************************************************/ -Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : +Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : _RR(this), RR(&_RR), _uPtr(uptr), @@ -72,26 +72,26 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : memset(_prngStream,0,sizeof(_prngStream)); _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - std::string idtmp(dataStoreGet("identity.secret")); + std::string idtmp(dataStoreGet(tptr,"identity.secret")); if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { TRACE("identity.secret not found, generating..."); RR->identity.generate(); idtmp = RR->identity.toString(true); - if (!dataStorePut("identity.secret",idtmp,true)) + if (!dataStorePut(tptr,"identity.secret",idtmp,true)) throw std::runtime_error("unable to write identity.secret"); } RR->publicIdentityStr = RR->identity.toString(false); RR->secretIdentityStr = RR->identity.toString(true); - idtmp = dataStoreGet("identity.public"); + idtmp = dataStoreGet(tptr,"identity.public"); if (idtmp != RR->publicIdentityStr) { - if (!dataStorePut("identity.public",RR->publicIdentityStr,false)) + if (!dataStorePut(tptr,"identity.public",RR->publicIdentityStr,false)) throw std::runtime_error("unable to write identity.public"); } try { RR->sw = new Switch(RR); RR->mc = new Multicaster(RR); - RR->topology = new Topology(RR); + RR->topology = new Topology(RR,tptr); RR->sa = new SelfAwareness(RR); } catch ( ... ) { delete RR->sa; @@ -101,7 +101,7 @@ Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : throw; } - postEvent(ZT_EVENT_UP); + postEvent(tptr,ZT_EVENT_UP); } Node::~Node() @@ -121,6 +121,7 @@ Node::~Node() } ZT_ResultCode Node::processWirePacket( + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -129,11 +130,12 @@ ZT_ResultCode Node::processWirePacket( volatile uint64_t *nextBackgroundTaskDeadline) { _now = now; - RR->sw->onRemotePacket(*(reinterpret_cast(localAddress)),*(reinterpret_cast(remoteAddress)),packetData,packetLength); + RR->sw->onRemotePacket(tptr,*(reinterpret_cast(localAddress)),*(reinterpret_cast(remoteAddress)),packetData,packetLength); return ZT_RESULT_OK; } ZT_ResultCode Node::processVirtualNetworkFrame( + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -147,7 +149,7 @@ ZT_ResultCode Node::processVirtualNetworkFrame( _now = now; SharedPtr nw(this->network(nwid)); if (nw) { - RR->sw->onLocalEthernet(nw,MAC(sourceMac),MAC(destMac),etherType,vlanId,frameData,frameLength); + RR->sw->onLocalEthernet(tptr,nw,MAC(sourceMac),MAC(destMac),etherType,vlanId,frameData,frameLength); return ZT_RESULT_OK; } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } @@ -156,9 +158,10 @@ ZT_ResultCode Node::processVirtualNetworkFrame( class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : lastReceiveFromUpstream(0), RR(renv), + _tPtr(tPtr), _upstreamsToContact(upstreamsToContact), _now(now), _bestCurrentUpstream(RR->topology->getUpstreamPeer()) @@ -176,21 +179,21 @@ public: // Upstreams must be pinged constantly over both IPv4 and IPv6 to allow // them to perform three way handshake introductions for both stacks. - if (!p->doPingAndKeepalive(_now,AF_INET)) { + if (!p->doPingAndKeepalive(_tPtr,_now,AF_INET)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET) { - p->sendHELLO(InetAddress(),addr,_now,0); + p->sendHELLO(_tPtr,InetAddress(),addr,_now,0); contacted = true; break; } } } else contacted = true; - if (!p->doPingAndKeepalive(_now,AF_INET6)) { + if (!p->doPingAndKeepalive(_tPtr,_now,AF_INET6)) { for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET6) { - p->sendHELLO(InetAddress(),addr,_now,0); + p->sendHELLO(_tPtr,InetAddress(),addr,_now,0); contacted = true; break; } @@ -200,24 +203,25 @@ public: if ((!contacted)&&(_bestCurrentUpstream)) { const SharedPtr up(_bestCurrentUpstream->getBestPath(_now,true)); if (up) - p->sendHELLO(up->localAddress(),up->address(),_now,up->nextOutgoingCounter()); + p->sendHELLO(_tPtr,up->localAddress(),up->address(),_now,up->nextOutgoingCounter()); } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); _upstreamsToContact.erase(p->address()); // erase from upstreams to contact so that we can WHOIS those that remain } else if (p->isActive(_now)) { - p->doPingAndKeepalive(_now,-1); + p->doPingAndKeepalive(_tPtr,_now,-1); } } private: const RuntimeEnvironment *RR; + void *_tPtr; Hashtable< Address,std::vector > &_upstreamsToContact; const uint64_t _now; const SharedPtr _bestCurrentUpstream; }; -ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) { _now = now; Mutex::Lock bl(_backgroundTasksLock); @@ -235,16 +239,16 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) needConfig.push_back(n->second); - n->second->sendUpdatesToMembers(); + n->second->sendUpdatesToMembers(tptr); } } for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) - (*n)->requestConfiguration(); + (*n)->requestConfiguration(tptr); // Do pings and keepalives Hashtable< Address,std::vector > upstreamsToContact; RR->topology->getUpstreamsToContact(upstreamsToContact); - _PingPeersThatNeedPing pfunc(RR,upstreamsToContact,now); + _PingPeersThatNeedPing pfunc(RR,tptr,upstreamsToContact,now); RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); // Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds) @@ -252,13 +256,13 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB Address *upstreamAddress = (Address *)0; std::vector *upstreamStableEndpoints = (std::vector *)0; while (i.next(upstreamAddress,upstreamStableEndpoints)) - RR->sw->requestWhois(*upstreamAddress); + RR->sw->requestWhois(tptr,*upstreamAddress); // Update online status, post status change as event const bool oldOnline = _online; _online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot())); if (oldOnline != _online) - postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE); + postEvent(tptr,_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -286,7 +290,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB *nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate } else { #endif - *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); + *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); #ifdef ZT_ENABLE_CLUSTER } #endif @@ -297,17 +301,17 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB return ZT_RESULT_OK; } -ZT_ResultCode Node::join(uint64_t nwid,void *uptr) +ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr) { Mutex::Lock _l(_networks_m); SharedPtr nw = _network(nwid); if(!nw) - _networks.push_back(std::pair< uint64_t,SharedPtr >(nwid,SharedPtr(new Network(RR,nwid,uptr)))); + _networks.push_back(std::pair< uint64_t,SharedPtr >(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr)))); std::sort(_networks.begin(),_networks.end()); // will sort by nwid since it's the first in a pair<> return ZT_RESULT_OK; } -ZT_ResultCode Node::leave(uint64_t nwid,void **uptr) +ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { std::vector< std::pair< uint64_t,SharedPtr > > newn; Mutex::Lock _l(_networks_m); @@ -324,11 +328,11 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr) return ZT_RESULT_OK; } -ZT_ResultCode Node::multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) +ZT_ResultCode Node::multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) { SharedPtr nw(this->network(nwid)); if (nw) { - nw->multicastSubscribe(MulticastGroup(MAC(multicastGroup),(uint32_t)(multicastAdi & 0xffffffff))); + nw->multicastSubscribe(tptr,MulticastGroup(MAC(multicastGroup),(uint32_t)(multicastAdi & 0xffffffff))); return ZT_RESULT_OK; } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } @@ -342,15 +346,15 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u } else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND; } -ZT_ResultCode Node::orbit(uint64_t moonWorldId,uint64_t moonSeed) +ZT_ResultCode Node::orbit(void *tptr,uint64_t moonWorldId,uint64_t moonSeed) { - RR->topology->addMoon(moonWorldId,Address(moonSeed)); + RR->topology->addMoon(tptr,moonWorldId,Address(moonSeed)); return ZT_RESULT_OK; } -ZT_ResultCode Node::deorbit(uint64_t moonWorldId) +ZT_ResultCode Node::deorbit(void *tptr,uint64_t moonWorldId) { - RR->topology->removeMoon(moonWorldId); + RR->topology->removeMoon(tptr,moonWorldId); return ZT_RESULT_OK; } @@ -465,7 +469,7 @@ void Node::clearLocalInterfaceAddresses() _directPaths.clear(); } -int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len) +int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) { try { if (RR->identity.address().toInt() != dest) { @@ -473,7 +477,7 @@ int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigne outp.append(typeId); outp.append(data,len); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send(tptr,outp,true); return 1; } } catch ( ... ) {} @@ -486,7 +490,7 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } -ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) +ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) { if (test->hopCount > 0) { try { @@ -516,7 +520,7 @@ ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback) for(unsigned int a=0;ahops[0].breadth;++a) { outp.newInitializationVector(); outp.setDestination(Address(test->hops[0].addresses[a])); - RR->sw->send(outp,true); + RR->sw->send(tptr,outp,true); } } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet @@ -616,13 +620,13 @@ void Node::clusterStatus(ZT_ClusterStatus *cs) /* Node methods used only within node/ */ /****************************************************************************/ -std::string Node::dataStoreGet(const char *name) +std::string Node::dataStoreGet(void *tPtr,const char *name) { char buf[1024]; std::string r; unsigned long olen = 0; do { - long n = _cb.dataStoreGetFunction(reinterpret_cast(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); + long n = _cb.dataStoreGetFunction(reinterpret_cast(this),_uPtr,tPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); if (n <= 0) return std::string(); r.append(buf,n); @@ -630,7 +634,7 @@ std::string Node::dataStoreGet(const char *name) return r; } -bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) +bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) { if (!Path::isAddressValidForPath(remoteAddress)) return false; @@ -650,7 +654,7 @@ bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddre } } - return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0) : true); + return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0) : true); } #ifdef ZT_TRACE @@ -728,7 +732,7 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de if (destination == RR->identity.address()) { SharedPtr n(network(nwid)); if (!n) return; - n->setConfiguration(nc,true); + n->setConfiguration((void *)0,nc,true); } else { Dictionary *dconf = new Dictionary(); try { @@ -762,7 +766,7 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de outp.append(sig.data,ZT_C25519_SIGNATURE_LEN); outp.compress(); - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); chunkIndex += chunkLen; } } @@ -779,7 +783,7 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev) if (destination == RR->identity.address()) { SharedPtr n(network(rev.networkId())); if (!n) return; - n->addCredential(RR->identity.address(),rev); + n->addCredential((void *)0,RR->identity.address(),rev); } else { Packet outp(destination,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS); outp.append((uint8_t)0x00); @@ -788,7 +792,7 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev) outp.append((uint16_t)1); rev.serialize(outp); outp.append((uint16_t)0); - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); } } @@ -823,7 +827,7 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des break; } outp.append(nwid); - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); } // else we can't send an ERROR() in response to nothing, so discard } @@ -835,11 +839,11 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des extern "C" { -enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) { *node = (ZT_Node *)0; try { - *node = reinterpret_cast(new ZeroTier::Node(uptr,callbacks,now)); + *node = reinterpret_cast(new ZeroTier::Node(uptr,tptr,callbacks,now)); return ZT_RESULT_OK; } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; @@ -859,6 +863,7 @@ void ZT_Node_delete(ZT_Node *node) enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -867,7 +872,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processWirePacket(now,localAddress,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processWirePacket(tptr,now,localAddress,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -877,6 +882,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -888,7 +894,7 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processVirtualNetworkFrame(now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processVirtualNetworkFrame(tptr,now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -896,10 +902,10 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( } } -enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processBackgroundTasks(now,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processBackgroundTasks(tptr,now,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -907,10 +913,10 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol } } -enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr) +enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr,void *tptr) { try { - return reinterpret_cast(node)->join(nwid,uptr); + return reinterpret_cast(node)->join(nwid,uptr,tptr); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -918,10 +924,10 @@ enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr) } } -enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr) +enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr,void *tptr) { try { - return reinterpret_cast(node)->leave(nwid,uptr); + return reinterpret_cast(node)->leave(nwid,uptr,tptr); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -929,10 +935,10 @@ enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid,void **uptr) } } -enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) +enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi) { try { - return reinterpret_cast(node)->multicastSubscribe(nwid,multicastGroup,multicastAdi); + return reinterpret_cast(node)->multicastSubscribe(tptr,nwid,multicastGroup,multicastAdi); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { @@ -951,19 +957,19 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint } } -enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed) +enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,uint64_t moonSeed) { try { - return reinterpret_cast(node)->orbit(moonWorldId,moonSeed); + return reinterpret_cast(node)->orbit(tptr,moonWorldId,moonSeed); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } } -ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId) +ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId) { try { - return reinterpret_cast(node)->deorbit(moonWorldId); + return reinterpret_cast(node)->deorbit(tptr,moonWorldId); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -1031,10 +1037,10 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node) } catch ( ... ) {} } -int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) +int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len) { try { - return reinterpret_cast(node)->sendUserMessage(dest,typeId,data,len); + return reinterpret_cast(node)->sendUserMessage(tptr,dest,typeId,data,len); } catch ( ... ) { return 0; } @@ -1047,10 +1053,10 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) +enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) { try { - return reinterpret_cast(node)->circuitTestBegin(test,reportCallback); + return reinterpret_cast(node)->circuitTestBegin(tptr,test,reportCallback); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } diff --git a/node/Node.hpp b/node/Node.hpp index 21eac617..03bd7a8c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -65,7 +65,7 @@ class World; class Node : public NetworkController::Sender { public: - Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); + Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); virtual ~Node(); // Get rid of alignment warnings on 32-bit Windows and possibly improve performance @@ -77,6 +77,7 @@ public: // Public API Functions ---------------------------------------------------- ZT_ResultCode processWirePacket( + void *tptr, uint64_t now, const struct sockaddr_storage *localAddress, const struct sockaddr_storage *remoteAddress, @@ -84,6 +85,7 @@ public: unsigned int packetLength, volatile uint64_t *nextBackgroundTaskDeadline); ZT_ResultCode processVirtualNetworkFrame( + void *tptr, uint64_t now, uint64_t nwid, uint64_t sourceMac, @@ -93,13 +95,13 @@ public: const void *frameData, unsigned int frameLength, volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode join(uint64_t nwid,void *uptr); - ZT_ResultCode leave(uint64_t nwid,void **uptr); - ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); + ZT_ResultCode processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); + ZT_ResultCode join(uint64_t nwid,void *uptr,void *tptr); + ZT_ResultCode leave(uint64_t nwid,void **uptr,void *tptr); + ZT_ResultCode multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); - ZT_ResultCode orbit(uint64_t moonWorldId,uint64_t moonSeed); - ZT_ResultCode deorbit(uint64_t moonWorldId); + ZT_ResultCode orbit(void *tptr,uint64_t moonWorldId,uint64_t moonSeed); + ZT_ResultCode deorbit(void *tptr,uint64_t moonWorldId); uint64_t address() const; void status(ZT_NodeStatus *status) const; ZT_PeerList *peers() const; @@ -108,9 +110,9 @@ public: void freeQueryResult(void *qr); int addLocalInterfaceAddress(const struct sockaddr_storage *addr); void clearLocalInterfaceAddresses(); - int sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len); + int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setNetconfMaster(void *networkControllerInstance); - ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); + ZT_ResultCode circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); void circuitTestEnd(ZT_CircuitTest *test); ZT_ResultCode clusterInit( unsigned int myId, @@ -132,11 +134,12 @@ public: inline uint64_t now() const throw() { return _now; } - inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) + inline bool putPacket(void *tPtr,const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { return (_cb.wirePacketSendFunction( reinterpret_cast(this), _uPtr, + tPtr, reinterpret_cast(&localAddress), reinterpret_cast(&addr), data, @@ -144,11 +147,12 @@ public: ttl) == 0); } - inline void putFrame(uint64_t nwid,void **nuptr,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) + inline void putFrame(void *tPtr,uint64_t nwid,void **nuptr,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { _cb.virtualNetworkFrameFunction( reinterpret_cast(this), _uPtr, + tPtr, nwid, nuptr, source.toInt(), @@ -191,14 +195,14 @@ public: return _directPaths; } - inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,name,data,len,(int)secure) == 0); } - inline bool dataStorePut(const char *name,const std::string &data,bool secure) { return dataStorePut(name,(const void *)data.data(),(unsigned int)data.length(),secure); } - inline void dataStoreDelete(const char *name) { _cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,name,(const void *)0,0,0); } - std::string dataStoreGet(const char *name); + inline bool dataStorePut(void *tPtr,const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,tPtr,name,data,len,(int)secure) == 0); } + inline bool dataStorePut(void *tPtr,const char *name,const std::string &data,bool secure) { return dataStorePut(tPtr,name,(const void *)data.data(),(unsigned int)data.length(),secure); } + inline void dataStoreDelete(void *tPtr,const char *name) { _cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,tPtr,name,(const void *)0,0,0); } + std::string dataStoreGet(void *tPtr,const char *name); - inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast(this),_uPtr,ev,md); } + inline void postEvent(void *tPtr,ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast(this),_uPtr,tPtr,ev,md); } - inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,nwid,nuptr,op,nc); } + inline int configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,tPtr,nwid,nuptr,op,nc); } inline bool online() const throw() { return _online; } @@ -206,8 +210,8 @@ public: void postTrace(const char *module,unsigned int line,const char *fmt,...); #endif - bool shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); - inline bool externalPathLookup(const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } + bool shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); + inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); void postCircuitTestReport(const ZT_CircuitTestReport *report); diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index d4cb87cb..285bfa5d 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -85,18 +85,18 @@ void OutboundMulticast::init( memcpy(_frameData,payload,_frameLen); } -void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr) +void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { const SharedPtr nw(RR->node->network(_nwid)); const Address toAddr2(toAddr); - if ((nw)&&(nw->filterOutgoingPacket(true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { + if ((nw)&&(nw->filterOutgoingPacket(tPtr,true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr2); RR->node->expectReplyTo(_packet.packetId()); Packet tmp(_packet); // make a copy of packet so as not to garble the original -- GitHub issue #461 - RR->sw->send(tmp,true); + RR->sw->send(tPtr,tmp,true); } } diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 6370d0d7..0ecf113f 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -99,33 +99,36 @@ public: * Just send without checking log * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param toAddr Destination address */ - void sendOnly(const RuntimeEnvironment *RR,const Address &toAddr); + void sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr); /** * Just send and log but do not check sent log * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param toAddr Destination address */ - inline void sendAndLog(const RuntimeEnvironment *RR,const Address &toAddr) + inline void sendAndLog(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { _alreadySentTo.push_back(toAddr); - sendOnly(RR,toAddr); + sendOnly(RR,tPtr,toAddr); } /** * Try to send this to a given peer if it hasn't been sent to them already * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param toAddr Destination address * @return True if address is new and packet was sent to switch, false if duplicate */ - inline bool sendIfNew(const RuntimeEnvironment *RR,const Address &toAddr) + inline bool sendIfNew(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr) { if (std::find(_alreadySentTo.begin(),_alreadySentTo.end(),toAddr) == _alreadySentTo.end()) { - sendAndLog(RR,toAddr); + sendAndLog(RR,tPtr,toAddr); return true; } else { return false; diff --git a/node/Path.cpp b/node/Path.cpp index 5592bacc..7366b56f 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -22,9 +22,9 @@ namespace ZeroTier { -bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now) +bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now) { - if (RR->node->putPacket(_localAddress,address(),data,len)) { + if (RR->node->putPacket(tPtr,_localAddress,address(),data,len)) { _lastOut = now; return true; } diff --git a/node/Path.hpp b/node/Path.hpp index 62f29c22..aef628d4 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -186,12 +186,13 @@ public: * Send a packet via this path (last out time is also updated) * * @param RR Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param data Packet data * @param len Packet length * @param now Current time * @return True if transport reported success */ - bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now); + bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now); /** * Manually update last sent time diff --git a/node/Peer.cpp b/node/Peer.cpp index fa3ce6c8..0cc23e33 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -68,6 +68,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident } void Peer::received( + void *tPtr, const SharedPtr &path, const unsigned int hops, const uint64_t packetId, @@ -161,7 +162,7 @@ void Peer::received( } } - if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(_id.address(),path->localAddress(),path->address())) ) { + if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); @@ -206,7 +207,7 @@ void Peer::received( #endif } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); + attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); path->sent(now); } } @@ -281,7 +282,7 @@ void Peer::received( if (count) { outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count); outp.armor(_key,true,path->nextOutgoingCounter()); - path->send(RR,outp.data(),outp.size(),now); + path->send(RR,tPtr,outp.data(),outp.size(),now); } } } @@ -299,7 +300,7 @@ bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const return false; } -bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) +bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) { Mutex::Lock _l(_paths_m); @@ -316,7 +317,7 @@ bool Peer::sendDirect(const void *data,unsigned int len,uint64_t now,bool forceE } if (bestp >= 0) { - return _paths[bestp].path->send(RR,data,len,now); + return _paths[bestp].path->send(RR,tPtr,data,len,now); } else { return false; } @@ -345,7 +346,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) } } -void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) +void Peer::sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -387,35 +388,35 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u if (atAddress) { outp.armor(_key,false,counter); // false == don't encrypt full payload, but add MAC - RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,localAddr,atAddress,outp.data(),outp.size()); } else { - RR->sw->send(outp,false); // false == don't encrypt full payload, but add MAC + RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC } } -void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) +void Peer::attemptToContactAt(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) { if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,true,counter); - RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,localAddr,atAddress,outp.data(),outp.size()); } else { - sendHELLO(localAddr,atAddress,now,counter); + sendHELLO(tPtr,localAddr,atAddress,now,counter); } } -void Peer::tryMemorizedPath(uint64_t now) +void Peer::tryMemorizedPath(void *tPtr,uint64_t now) { if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) { _lastTriedMemorizedPath = now; InetAddress mp; - if (RR->node->externalPathLookup(_id.address(),-1,mp)) - attemptToContactAt(InetAddress(),mp,now,true,0); + if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp)) + attemptToContactAt(tPtr,InetAddress(),mp,now,true,0); } } -bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) +bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); @@ -433,7 +434,7 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily) if (bestp >= 0) { if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { - attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); + attemptToContactAt(tPtr,_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); _paths[bestp].path->sent(now); } return true; @@ -452,12 +453,12 @@ bool Peer::hasActiveDirectPath(uint64_t now) const return false; } -void Peer::resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) +void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) { Mutex::Lock _l(_paths_m); for(unsigned int p=0;p<_numPaths;++p) { if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - attemptToContactAt(_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); + attemptToContactAt(tPtr,_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); _paths[p].path->sent(now); _paths[p].lastReceive = 0; // path will not be used unless it speaks again } diff --git a/node/Peer.hpp b/node/Peer.hpp index 72040b1d..41836410 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -84,6 +84,7 @@ public: * This is called by the decode pipe when a packet is proven to be authentic * and appears to be valid. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param path Path over which packet was received * @param hops ZeroTier (not IP) hops * @param packetId Packet ID @@ -93,6 +94,7 @@ public: * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established */ void received( + void *tPtr, const SharedPtr &path, const unsigned int hops, const uint64_t packetId, @@ -125,13 +127,14 @@ public: /** * Send via best direct path * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param data Packet data * @param len Packet length * @param now Current time * @param forceEvenIfDead If true, send even if the path is not 'alive' * @return True if we actually sent something */ - bool sendDirect(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); + bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); /** * Get the best current direct path @@ -147,41 +150,47 @@ public: * * No statistics or sent times are updated here. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localAddr Local address * @param atAddress Destination address * @param now Current time * @param counter Outgoing packet counter */ - void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter); + void sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter); /** * Send ECHO (or HELLO for older peers) to this peer at the given address * * No statistics or sent times are updated here. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localAddr Local address * @param atAddress Destination address * @param now Current time * @param sendFullHello If true, always send a full HELLO instead of just an ECHO * @param counter Outgoing packet counter */ - void attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); + void attemptToContactAt(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); /** * Try a memorized or statically defined path if any are known * * Under the hood this is done periodically based on ZT_TRY_MEMORIZED_PATH_INTERVAL. + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param now Current time */ - void tryMemorizedPath(uint64_t now); + void tryMemorizedPath(void *tPtr,uint64_t now); /** * Send pings or keepalives depending on configured timeouts * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param inetAddressFamily Keep this address family alive, or -1 for any * @return True if we have at least one direct path of the given family (or any if family is -1) */ - bool doPingAndKeepalive(uint64_t now,int inetAddressFamily); + bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); /** * @param now Current time @@ -195,11 +204,12 @@ public: * Resetting a path involves sending an ECHO to it and then deactivating * it until or unless it responds. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param scope IP scope * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - void resetWithinScope(InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); + void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); /** * Get most recently active path addresses for IPv4 and/or IPv6 diff --git a/node/Revocation.cpp b/node/Revocation.cpp index 420476a4..bab5653c 100644 --- a/node/Revocation.cpp +++ b/node/Revocation.cpp @@ -25,13 +25,13 @@ namespace ZeroTier { -int Revocation::verify(const RuntimeEnvironment *RR) const +int Revocation::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } try { diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 93c55112..8b9ce6dd 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -113,9 +113,10 @@ public: * Verify this revocation's signature * * @param RR Runtime environment to provide for peer lookup, etc. + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or chain */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template inline void serialize(Buffer &b,const bool forSign = false) const diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index e84b7b65..cba84cdc 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -40,15 +40,17 @@ namespace ZeroTier { class _ResetWithinScope { public: - _ResetWithinScope(uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : + _ResetWithinScope(void *tPtr,uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : _now(now), + _tPtr(tPtr), _family(inetAddressFamily), _scope(scope) {} - inline void operator()(Topology &t,const SharedPtr &p) { p->resetWithinScope(_scope,_family,_now); } + inline void operator()(Topology &t,const SharedPtr &p) { p->resetWithinScope(_tPtr,_scope,_family,_now); } private: uint64_t _now; + void *_tPtr; int _family; InetAddress::IpScope _scope; }; @@ -59,7 +61,7 @@ SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) : { } -void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) +void SelfAwareness::iam(void *tPtr,const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) { const InetAddress::IpScope scope = myPhysicalAddress.ipScope(); @@ -91,7 +93,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &receivedOnLoc } // Reset all paths within this scope and address family - _ResetWithinScope rset(now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope); + _ResetWithinScope rset(tPtr,now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope); RR->topology->eachPeer<_ResetWithinScope &>(rset); } else { // Otherwise just update DB to use to determine external surface info diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index 4bdafeb2..c1db0c84 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -47,7 +47,7 @@ public: * @param trusted True if this peer is trusted as an authority to inform us of external address changes * @param now Current time */ - void iam(const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); + void iam(void *tPtr,const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); /** * Clean up database periodically diff --git a/node/Switch.cpp b/node/Switch.cpp index aab2e7ff..62674472 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -64,7 +64,7 @@ Switch::Switch(const RuntimeEnvironment *renv) : { } -void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len) +void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len) { try { const uint64_t now = RR->node->now(); @@ -81,15 +81,15 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from const Address beaconAddr(reinterpret_cast(data) + 8,5); if (beaconAddr == RR->identity.address()) return; - if (!RR->node->shouldUsePathForZeroTierTraffic(beaconAddr,localAddr,fromAddr)) + if (!RR->node->shouldUsePathForZeroTierTraffic(tPtr,beaconAddr,localAddr,fromAddr)) return; - const SharedPtr peer(RR->topology->getPeer(beaconAddr)); + const SharedPtr peer(RR->topology->getPeer(tPtr,beaconAddr)); if (peer) { // we'll only respond to beacons from known peers if ((now - _lastBeaconResponse) >= 2500) { // limit rate of responses _lastBeaconResponse = now; Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP); outp.armor(peer->key(),true,path->nextOutgoingCounter()); - path->send(RR,outp.data(),outp.size(),now); + path->send(RR,tPtr,outp.data(),outp.size(),now); } } @@ -115,8 +115,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Note: we don't bother initiating NAT-t for fragments, since heads will set that off. // It wouldn't hurt anything, just redundant and unnecessary. - SharedPtr relayTo = RR->topology->getPeer(destination); - if ((!relayTo)||(!relayTo->sendDirect(fragment.data(),fragment.size(),now,false))) { + SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); + if ((!relayTo)||(!relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,false))) { #ifdef ZT_ENABLE_CLUSTER if ((RR->cluster)&&(!isClusterFrontplane)) { RR->cluster->relayViaCluster(Address(),destination,fragment.data(),fragment.size(),false); @@ -127,7 +127,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from // Don't know peer or no direct path -- so relay via someone upstream relayTo = RR->topology->getUpstreamPeer(); if (relayTo) - relayTo->sendDirect(fragment.data(),fragment.size(),now,true); + relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,true); } } else { TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str()); @@ -171,7 +171,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from for(unsigned int f=1;ffrag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); - if (rq->frag0.tryDecode(RR)) { + if (rq->frag0.tryDecode(RR,tPtr)) { rq->timestamp = 0; // packet decoded, free entry } else { rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something @@ -212,8 +212,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from packet.incrementHops(); #endif - SharedPtr relayTo = RR->topology->getPeer(destination); - if ((relayTo)&&(relayTo->sendDirect(packet.data(),packet.size(),now,false))) { + SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); + if ((relayTo)&&(relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,false))) { if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { // don't send RENDEZVOUS for cluster frontplane relays const InetAddress *hintToSource = (InetAddress *)0; const InetAddress *hintToDest = (InetAddress *)0; @@ -222,7 +222,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from InetAddress sourceV4,sourceV6; relayTo->getRendezvousAddresses(now,destV4,destV6); - const SharedPtr sourcePeer(RR->topology->getPeer(source)); + const SharedPtr sourcePeer(RR->topology->getPeer(tPtr,source)); if (sourcePeer) { sourcePeer->getRendezvousAddresses(now,sourceV4,sourceV6); if ((destV6)&&(sourceV6)) { @@ -249,7 +249,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from outp.append((uint8_t)4); outp.append(hintToSource->rawIpData(),4); } - send(outp,true); + send(tPtr,outp,true); } else { Packet outp(destination,RR->identity.address(),Packet::VERB_RENDEZVOUS); outp.append((uint8_t)0); @@ -262,7 +262,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from outp.append((uint8_t)4); outp.append(hintToDest->rawIpData(),4); } - send(outp,true); + send(tPtr,outp,true); } ++alt; } @@ -278,7 +278,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from #endif relayTo = RR->topology->getUpstreamPeer(&source,1,true); if (relayTo) - relayTo->sendDirect(packet.data(),packet.size(),now,true); + relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true); } } else { TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet.source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str()); @@ -321,7 +321,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from for(unsigned int f=1;ftotalFragments;++f) rq->frag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); - if (rq->frag0.tryDecode(RR)) { + if (rq->frag0.tryDecode(RR,tPtr)) { rq->timestamp = 0; // packet decoded, free entry } else { rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something @@ -334,7 +334,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } else { // Packet is unfragmented, so just process it IncomingPacket packet(data,len,path,now); - if (!packet.tryDecode(RR)) { + if (!packet.tryDecode(RR,tPtr)) { Mutex::Lock _l(_rxQueue_m); RXQueueEntry *rq = &(_rxQueue[ZT_RX_QUEUE_SIZE - 1]); unsigned long i = ZT_RX_QUEUE_SIZE - 1; @@ -362,7 +362,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from } } -void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) +void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { if (!network->hasConfig()) return; @@ -474,7 +474,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c adv[42] = (checksum >> 8) & 0xff; adv[43] = checksum & 0xff; - RR->node->putFrame(network->id(),network->userPtr(),peerMac,from,ZT_ETHERTYPE_IPV6,0,adv,72); + RR->node->putFrame(tPtr,network->id(),network->userPtr(),peerMac,from,ZT_ETHERTYPE_IPV6,0,adv,72); return; // NDP emulation done. We have forged a "fake" reply, so no need to send actual NDP query. } // else no NDP emulation } // else no NDP emulation @@ -491,17 +491,18 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c * multicast addresses on bridge interfaces and subscribing each slave. * But in that case this does no harm, as the sets are just merged. */ if (fromBridged) - network->learnBridgedMulticastGroup(multicastGroup,RR->node->now()); + network->learnBridgedMulticastGroup(tPtr,multicastGroup,RR->node->now()); //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),multicastGroup.toString().c_str(),etherTypeName(etherType),len); // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. - if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } RR->mc->send( + tPtr, network->config().multicastLimit, RR->node->now(), network->id(), @@ -514,14 +515,14 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c len); } else if (to == network->mac()) { // Destination is this node, so just reinject it - RR->node->putFrame(network->id(),network->userPtr(),from,to,etherType,vlanId,data,len); + RR->node->putFrame(tPtr,network->id(),network->userPtr(),from,to,etherType,vlanId,data,len); } else if (to[0] == MAC::firstOctetForNetwork(network->id())) { // Destination is another ZeroTier peer on the same network Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this - SharedPtr toPeer(RR->topology->getPeer(toZT)); + SharedPtr toPeer(RR->topology->getPeer(tPtr,toZT)); - if (!network->filterOutgoingPacket(false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } @@ -536,7 +537,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(outp,true); + send(tPtr,outp,true); } else { Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME); outp.append(network->id()); @@ -544,7 +545,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(outp,true); + send(tPtr,outp,true); } //TRACE("%.16llx: UNICAST: %s -> %s etherType==%s(%.4x) vlanId==%u len==%u fromBridged==%d includeCom==%d",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),etherType,vlanId,len,(int)fromBridged,(int)includeCom); @@ -554,7 +555,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c // We filter with a NULL destination ZeroTier address first. Filtrations // for each ZT destination are also done below. This is the same rationale // and design as for multicast. - if (!network->filterOutgoingPacket(false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); return; } @@ -592,7 +593,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } for(unsigned int b=0;bfilterOutgoingPacket(true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId)) { + if (network->filterOutgoingPacket(tPtr,true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId)) { Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(network->id()); outp.append((uint8_t)0x00); @@ -602,7 +603,7 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c outp.append(data,len); if (!network->config().disableCompression()) outp.compress(); - send(outp,true); + send(tPtr,outp,true); } else { TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); } @@ -610,20 +611,20 @@ void Switch::onLocalEthernet(const SharedPtr &network,const MAC &from,c } } -void Switch::send(Packet &packet,bool encrypt) +void Switch::send(void *tPtr,Packet &packet,bool encrypt) { if (packet.destination() == RR->identity.address()) { TRACE("BUG: caught attempt to send() to self, ignored"); return; } - if (!_trySend(packet,encrypt)) { + if (!_trySend(tPtr,packet,encrypt)) { Mutex::Lock _l(_txQueue_m); _txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt)); } } -void Switch::requestWhois(const Address &addr) +void Switch::requestWhois(void *tPtr,const Address &addr) { #ifdef ZT_TRACE if (addr == RR->identity.address()) { @@ -644,10 +645,10 @@ void Switch::requestWhois(const Address &addr) } } if (inserted) - _sendWhoisRequest(addr,(const Address *)0,0); + _sendWhoisRequest(tPtr,addr,(const Address *)0,0); } -void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) +void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) { { // cancel pending WHOIS since we now know this peer Mutex::Lock _l(_outstandingWhoisRequests_m); @@ -660,7 +661,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) while (i) { RXQueueEntry *rq = &(_rxQueue[--i]); if ((rq->timestamp)&&(rq->complete)) { - if (rq->frag0.tryDecode(RR)) + if (rq->frag0.tryDecode(RR,tPtr)) rq->timestamp = 0; } } @@ -670,7 +671,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { if (txi->dest == peer->address()) { - if (_trySend(txi->packet,txi->encrypt)) + if (_trySend(tPtr,txi->packet,txi->encrypt)) _txQueue.erase(txi++); else ++txi; } else ++txi; @@ -678,7 +679,7 @@ void Switch::doAnythingWaitingForPeer(const SharedPtr &peer) } } -unsigned long Switch::doTimerTasks(uint64_t now) +unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) { unsigned long nextDelay = 0xffffffff; // ceiling delay, caller will cap to minimum @@ -695,7 +696,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) _outstandingWhoisRequests.erase(*a); } else { r->lastSent = now; - r->peersConsulted[r->retries] = _sendWhoisRequest(*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); + r->peersConsulted[r->retries] = _sendWhoisRequest(tPtr,*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); TRACE("WHOIS %s (retry %u)",a->toString().c_str(),r->retries); ++r->retries; nextDelay = std::min(nextDelay,(unsigned long)ZT_WHOIS_RETRY_DELAY); @@ -709,7 +710,7 @@ unsigned long Switch::doTimerTasks(uint64_t now) { // Time out TX queue packets that never got WHOIS lookups or other info. Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { - if (_trySend(txi->packet,txi->encrypt)) + if (_trySend(tPtr,txi->packet,txi->encrypt)) _txQueue.erase(txi++); else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { TRACE("TX %s -> %s timed out",txi->packet.source().toString().c_str(),txi->packet.destination().toString().c_str()); @@ -743,19 +744,19 @@ bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address return false; } -Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) +Address Switch::_sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) { SharedPtr upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); if (upstream) { Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); addr.appendTo(outp); RR->node->expectReplyTo(outp.packetId()); - send(outp,true); + send(tPtr,outp,true); } return Address(); } -bool Switch::_trySend(Packet &packet,bool encrypt) +bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) { SharedPtr viaPath; const uint64_t now = RR->node->now(); @@ -769,7 +770,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) clusterMostRecentMemberId = RR->cluster->checkSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); #endif - const SharedPtr peer(RR->topology->getPeer(destination)); + const SharedPtr peer(RR->topology->getPeer(tPtr,destination)); if (peer) { /* First get the best path, and if it's dead (and this is not a root) * we attempt to re-activate that path but this packet will flow @@ -784,7 +785,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) if ((clusterMostRecentMemberId < 0)||(viaPath->lastIn() > clusterMostRecentTs)) { #endif if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); + peer->attemptToContactAt(tPtr,viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); viaPath->sent(now); } #ifdef ZT_ENABLE_CLUSTER @@ -801,7 +802,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #else if (!viaPath) { #endif - peer->tryMemorizedPath(now); // periodically attempt memorized or statically defined paths, if any are known + peer->tryMemorizedPath(tPtr,now); // periodically attempt memorized or statically defined paths, if any are known const SharedPtr relay(RR->topology->getUpstreamPeer()); if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { if (!(viaPath = peer->getBestPath(now,true))) @@ -816,7 +817,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #ifdef ZT_ENABLE_CLUSTER if (clusterMostRecentMemberId < 0) { #else - requestWhois(destination); + requestWhois(tPtr,destination); return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly #endif #ifdef ZT_ENABLE_CLUSTER @@ -844,9 +845,9 @@ bool Switch::_trySend(Packet &packet,bool encrypt) #endif #ifdef ZT_ENABLE_CLUSTER - if ( ((viaPath)&&(viaPath->send(RR,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { + if ( ((viaPath)&&(viaPath->send(RR,tPtr,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { #else - if (viaPath->send(RR,packet.data(),chunkSize,now)) { + if (viaPath->send(RR,tPtr,packet.data(),chunkSize,now)) { #endif if (chunkSize < packet.size()) { // Too big for one packet, fragment the rest @@ -866,7 +867,7 @@ bool Switch::_trySend(Packet &packet,bool encrypt) else if (clusterMostRecentMemberId >= 0) RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,frag.data(),frag.size()); #else - viaPath->send(RR,frag.data(),frag.size(),now); + viaPath->send(RR,tPtr,frag.data(),frag.size(),now); #endif fragStart += chunkSize; remaining -= chunkSize; diff --git a/node/Switch.hpp b/node/Switch.hpp index 9245c036..ff350934 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -59,16 +59,18 @@ public: /** * Called when a packet is received from the real network * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localAddr Local interface address * @param fromAddr Internet IP address of origin * @param data Packet data * @param len Packet length */ - void onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len); + void onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len); /** * Called when a packet comes from a local Ethernet tap * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param network Which network's TAP did this packet come from? * @param from Originating MAC address * @param to Destination MAC address @@ -77,7 +79,7 @@ public: * @param data Ethernet payload * @param len Frame length */ - void onLocalEthernet(const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); + void onLocalEthernet(void *tPtr,const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); /** * Send a packet to a ZeroTier address (destination in packet) @@ -91,26 +93,29 @@ public: * Needless to say, the packet's source must be this node. Otherwise it * won't be encrypted right. (This is not used for relaying.) * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param packet Packet to send (buffer may be modified) * @param encrypt Encrypt packet payload? (always true except for HELLO) */ - void send(Packet &packet,bool encrypt); + void send(void *tPtr,Packet &packet,bool encrypt); /** * Request WHOIS on a given address * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param addr Address to look up */ - void requestWhois(const Address &addr); + void requestWhois(void *tPtr,const Address &addr); /** * Run any processes that are waiting for this peer's identity * * Called when we learn of a peer's identity from HELLO, OK(WHOIS), etc. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param peer New peer */ - void doAnythingWaitingForPeer(const SharedPtr &peer); + void doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer); /** * Perform retries and other periodic timer tasks @@ -118,15 +123,16 @@ public: * This can return a very long delay if there are no pending timer * tasks. The caller should cap this comparatively vs. other values. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @return Number of milliseconds until doTimerTasks() should be run again */ - unsigned long doTimerTasks(uint64_t now); + unsigned long doTimerTasks(void *tPtr,uint64_t now); private: bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination); - Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); - bool _trySend(Packet &packet,bool encrypt); // packet is modified if return is true + Address _sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); + bool _trySend(void *tPtr,Packet &packet,bool encrypt); // packet is modified if return is true const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; diff --git a/node/Tag.cpp b/node/Tag.cpp index eb4026bc..3f924da1 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -25,13 +25,13 @@ namespace ZeroTier { -int Tag::verify(const RuntimeEnvironment *RR) const +int Tag::verify(const RuntimeEnvironment *RR,void *tPtr) const { if ((!_signedBy)||(_signedBy != Network::controllerFor(_networkId))) return -1; - const Identity id(RR->topology->getIdentity(_signedBy)); + const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(_signedBy); + RR->sw->requestWhois(tPtr,_signedBy); return 1; } try { diff --git a/node/Tag.hpp b/node/Tag.hpp index 146e8da9..38085906 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -105,9 +105,10 @@ public: * Check this tag's signature * * @param RR Runtime environment to allow identity lookup for signedBy + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or tag */ - int verify(const RuntimeEnvironment *RR) const; + int verify(const RuntimeEnvironment *RR,void *tPtr) const; template inline void serialize(Buffer &b,const bool forSign = false) const diff --git a/node/Topology.cpp b/node/Topology.cpp index 21547cd2..a1d37332 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -55,19 +55,19 @@ namespace ZeroTier { #define ZT_DEFAULT_WORLD_LENGTH 634 static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x52,0x3c,0x32,0x50,0x1a,0xb8,0xb3,0x88,0xa4,0x69,0x22,0x14,0x91,0xaa,0x9a,0xcd,0x66,0xcc,0x76,0x4c,0xde,0xfd,0x56,0x03,0x9f,0x10,0x67,0xae,0x15,0xe6,0x9c,0x6f,0xb4,0x2d,0x7b,0x55,0x33,0x0e,0x3f,0xda,0xac,0x52,0x9c,0x07,0x92,0xfd,0x73,0x40,0xa6,0xaa,0x21,0xab,0xa8,0xa4,0x89,0xfd,0xae,0xa4,0x4a,0x39,0xbf,0x2d,0x00,0x65,0x9a,0xc9,0xc8,0x18,0xeb,0x4a,0xf7,0x86,0xa8,0x40,0xd6,0x52,0xea,0xae,0x9e,0x7a,0xbf,0x4c,0x97,0x66,0xab,0x2d,0x6f,0xaf,0xc9,0x2b,0x3a,0xff,0xed,0xd6,0x30,0x3e,0xc4,0x6a,0x65,0xf2,0xbd,0x83,0x52,0xf5,0x40,0xe9,0xcc,0x0d,0x6e,0x89,0x3f,0x9a,0xa0,0xb8,0xdf,0x42,0xd2,0x2f,0x84,0xe6,0x03,0x26,0x0f,0xa8,0xe3,0xcc,0x05,0x05,0x03,0xef,0x12,0x80,0x0d,0xce,0x3e,0xb6,0x58,0x3b,0x1f,0xa8,0xad,0xc7,0x25,0xf9,0x43,0x71,0xa7,0x5c,0x9a,0xc7,0xe1,0xa3,0xb8,0x88,0xd0,0x71,0x6c,0x94,0x99,0x73,0x41,0x0b,0x1b,0x48,0x84,0x02,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x0c,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9a,0x42,0xc5,0x21,0x27,0x09,0x06,0x2c,0x0f,0xf8,0x50,0x01,0x54,0x01,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x33,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x6b,0xaa,0xc5,0x0e,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x00,0x01,0x00,0x20,0x00,0x00,0x00,0x00,0x02,0x00,0xe0,0x01,0x27,0x09,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0xb7,0x40,0x01,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x0c,0x04,0x2d,0x20,0xc6,0x82,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x64,0x00,0x81,0xc3,0x54,0x00,0x00,0xff,0xfe,0x18,0x1d,0x61,0x27,0x09,0x04,0x2e,0x65,0xa0,0xf9,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x03,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x6a,0x30,0x01,0x27,0x09,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x68,0x00,0x83,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x64,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09}; -Topology::Topology(const RuntimeEnvironment *renv) : +Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : RR(renv), _trustedPathCount(0), _amRoot(false) { try { World cachedPlanet; - std::string buf(RR->node->dataStoreGet("planet")); + std::string buf(RR->node->dataStoreGet(tPtr,"planet")); if (buf.length() > 0) { Buffer dswtmp(buf.data(),(unsigned int)buf.length()); cachedPlanet.deserialize(dswtmp,0); } - addWorld(cachedPlanet,false); + addWorld(tPtr,cachedPlanet,false); } catch ( ... ) {} World defaultPlanet; @@ -75,10 +75,10 @@ Topology::Topology(const RuntimeEnvironment *renv) : Buffer wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH); defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top } - addWorld(defaultPlanet,false); + addWorld(tPtr,defaultPlanet,false); } -SharedPtr Topology::addPeer(const SharedPtr &peer) +SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) { #ifdef ZT_TRACE if ((!peer)||(peer->address() == RR->identity.address())) { @@ -98,12 +98,12 @@ SharedPtr Topology::addPeer(const SharedPtr &peer) np = hp; } - saveIdentity(np->identity()); + saveIdentity(tPtr,np->identity()); return np; } -SharedPtr Topology::getPeer(const Address &zta) +SharedPtr Topology::getPeer(void *tPtr,const Address &zta) { if (zta == RR->identity.address()) { TRACE("BUG: ignored attempt to getPeer() for self, returned NULL"); @@ -118,7 +118,7 @@ SharedPtr Topology::getPeer(const Address &zta) } try { - Identity id(_getIdentity(zta)); + Identity id(_getIdentity(tPtr,zta)); if (id) { SharedPtr np(new Peer(RR,RR->identity,id)); { @@ -134,7 +134,7 @@ SharedPtr Topology::getPeer(const Address &zta) return SharedPtr(); } -Identity Topology::getIdentity(const Address &zta) +Identity Topology::getIdentity(void *tPtr,const Address &zta) { if (zta == RR->identity.address()) { return RR->identity; @@ -144,15 +144,15 @@ Identity Topology::getIdentity(const Address &zta) if (ap) return (*ap)->identity(); } - return _getIdentity(zta); + return _getIdentity(tPtr,zta); } -void Topology::saveIdentity(const Identity &id) +void Topology::saveIdentity(void *tPtr,const Identity &id) { if (id) { char p[128]; Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)id.address().toInt()); - RR->node->dataStorePut(p,id.toString(false),false); + RR->node->dataStorePut(tPtr,p,id.toString(false),false); } } @@ -264,7 +264,7 @@ bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipa return false; } -bool Topology::addWorld(const World &newWorld,bool alwaysAcceptNew) +bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew) { if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON)) return false; @@ -328,29 +328,29 @@ bool Topology::addWorld(const World &newWorld,bool alwaysAcceptNew) try { Buffer dswtmp; existing->serialize(dswtmp,false); - RR->node->dataStorePut(savePath,dswtmp.data(),dswtmp.size(),false); + RR->node->dataStorePut(tPtr,savePath,dswtmp.data(),dswtmp.size(),false); } catch ( ... ) { - RR->node->dataStoreDelete(savePath); + RR->node->dataStoreDelete(tPtr,savePath); } - _memoizeUpstreams(); + _memoizeUpstreams(tPtr); return true; } -void Topology::addMoon(const uint64_t id,const Address &seed) +void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed) { char savePath[64]; Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); try { - std::string moonBin(RR->node->dataStoreGet(savePath)); + std::string moonBin(RR->node->dataStoreGet(tPtr,savePath)); if (moonBin.length() > 1) { Buffer wtmp(moonBin.data(),(unsigned int)moonBin.length()); World w; w.deserialize(wtmp); if ((w.type() == World::TYPE_MOON)&&(w.id() == id)) { - addWorld(w,true); + addWorld(tPtr,w,true); return; } } @@ -363,7 +363,7 @@ void Topology::addMoon(const uint64_t id,const Address &seed) } } -void Topology::removeMoon(const uint64_t id) +void Topology::removeMoon(void *tPtr,const uint64_t id) { Mutex::Lock _l1(_upstreams_m); Mutex::Lock _l2(_peers_m); @@ -375,7 +375,7 @@ void Topology::removeMoon(const uint64_t id) } else { char savePath[64]; Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); - RR->node->dataStoreDelete(savePath); + RR->node->dataStoreDelete(tPtr,savePath); } } _moons.swap(nm); @@ -387,7 +387,7 @@ void Topology::removeMoon(const uint64_t id) } _moonSeeds.swap(cm); - _memoizeUpstreams(); + _memoizeUpstreams(tPtr); } void Topology::clean(uint64_t now) @@ -415,11 +415,11 @@ void Topology::clean(uint64_t now) } } -Identity Topology::_getIdentity(const Address &zta) +Identity Topology::_getIdentity(void *tPtr,const Address &zta) { char p[128]; Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)zta.toInt()); - std::string ids(RR->node->dataStoreGet(p)); + std::string ids(RR->node->dataStoreGet(tPtr,p)); if (ids.length() > 0) { try { return Identity(ids); @@ -428,7 +428,7 @@ Identity Topology::_getIdentity(const Address &zta) return Identity(); } -void Topology::_memoizeUpstreams() +void Topology::_memoizeUpstreams(void *tPtr) { // assumes _upstreams_m and _peers_m are locked _upstreamAddresses.clear(); @@ -442,7 +442,7 @@ void Topology::_memoizeUpstreams() SharedPtr &hp = _peers[i->identity.address()]; if (!hp) { hp = new Peer(RR,RR->identity,i->identity); - saveIdentity(i->identity); + saveIdentity(tPtr,i->identity); } } } @@ -456,7 +456,7 @@ void Topology::_memoizeUpstreams() SharedPtr &hp = _peers[i->identity.address()]; if (!hp) { hp = new Peer(RR,RR->identity,i->identity); - saveIdentity(i->identity); + saveIdentity(tPtr,i->identity); } } } diff --git a/node/Topology.hpp b/node/Topology.hpp index e21747c8..4870ab5e 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -50,7 +50,7 @@ class RuntimeEnvironment; class Topology { public: - Topology(const RuntimeEnvironment *renv); + Topology(const RuntimeEnvironment *renv,void *tPtr); /** * Add a peer to database @@ -58,18 +58,20 @@ public: * This will not replace existing peers. In that case the existing peer * record is returned. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param peer Peer to add * @return New or existing peer (should replace 'peer') */ - SharedPtr addPeer(const SharedPtr &peer); + SharedPtr addPeer(void *tPtr,const SharedPtr &peer); /** * Get a peer from its address * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param zta ZeroTier address of peer * @return Peer or NULL if not found */ - SharedPtr getPeer(const Address &zta); + SharedPtr getPeer(void *tPtr,const Address &zta); /** * Get a peer only if it is presently in memory (no disk cache) @@ -109,10 +111,11 @@ public: /** * Get the identity of a peer * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param zta ZeroTier address of peer * @return Identity or NULL Identity if not found */ - Identity getIdentity(const Address &zta); + Identity getIdentity(void *tPtr,const Address &zta); /** * Cache an identity @@ -120,9 +123,10 @@ public: * This is done automatically on addPeer(), and so is only useful for * cluster identity replication. * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param id Identity to cache */ - void saveIdentity(const Identity &id); + void saveIdentity(void *tPtr,const Identity &id); /** * Get the current best upstream peer @@ -267,11 +271,12 @@ public: /** * Validate new world and update if newer and signature is okay * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param newWorld A new or updated planet or moon to learn * @param alwaysAcceptNew If true, always accept new moons even if we're not waiting for one * @return True if it was valid and newer than current (or totally new for moons) */ - bool addWorld(const World &newWorld,bool alwaysAcceptNew); + bool addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew); /** * Add a moon @@ -282,14 +287,15 @@ public: * @param id Moon ID * @param seed If non-NULL, an address of any member of the moon to contact */ - void addMoon(const uint64_t id,const Address &seed); + void addMoon(void *tPtr,const uint64_t id,const Address &seed); /** * Remove a moon * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param id Moon's world ID */ - void removeMoon(const uint64_t id); + void removeMoon(void *tPtr,const uint64_t id); /** * Clean and flush database @@ -420,8 +426,8 @@ public: } private: - Identity _getIdentity(const Address &zta); - void _memoizeUpstreams(); + Identity _getIdentity(void *tPtr,const Address &zta); + void _memoizeUpstreams(void *tPtr); const RuntimeEnvironment *const RR; diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index 0e1ada6b..62fabc48 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -71,7 +71,7 @@ BSDEthernetTap::BSDEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) : _handler(handler), _arg(arg), @@ -460,8 +460,7 @@ void BSDEthernetTap::threadMain() to.setTo(getBuf,6); from.setTo(getBuf + 6,6); unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); - // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); } r = 0; diff --git a/osdep/BSDEthernetTap.hpp b/osdep/BSDEthernetTap.hpp index 1bb48d31..8c6314db 100644 --- a/osdep/BSDEthernetTap.hpp +++ b/osdep/BSDEthernetTap.hpp @@ -43,7 +43,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~BSDEthernetTap(); @@ -62,7 +62,7 @@ public: throw(); private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; Thread _thread; diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index e7fe657f..c4b978e7 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -62,7 +62,7 @@ LinuxEthernetTap::LinuxEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) : _handler(handler), _arg(arg), @@ -470,7 +470,7 @@ void LinuxEthernetTap::threadMain() from.setTo(getBuf + 6,6); unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); } r = 0; diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp index 7dd7e01d..a2a00a79 100644 --- a/osdep/LinuxEthernetTap.hpp +++ b/osdep/LinuxEthernetTap.hpp @@ -44,7 +44,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~LinuxEthernetTap(); @@ -66,7 +66,7 @@ public: throw(); private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; Thread _thread; diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index b3580929..35eac05a 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -314,7 +314,7 @@ OSXEthernetTap::OSXEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len), void *arg) : _handler(handler), _arg(arg), @@ -646,7 +646,7 @@ void OSXEthernetTap::threadMain() from.setTo(getBuf + 6,6); unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]); // TODO: VLAN support - _handler(_arg,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14); } r = 0; diff --git a/osdep/OSXEthernetTap.hpp b/osdep/OSXEthernetTap.hpp index de48f9a4..5a96c210 100644 --- a/osdep/OSXEthernetTap.hpp +++ b/osdep/OSXEthernetTap.hpp @@ -48,7 +48,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~OSXEthernetTap(); @@ -67,7 +67,7 @@ public: throw(); private: - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; Thread _thread; diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index 8ee088bb..79b9d35e 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -456,7 +456,7 @@ WindowsEthernetTap::WindowsEthernetTap( unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) : _handler(handler), _arg(arg), @@ -1058,8 +1058,7 @@ void WindowsEthernetTap::threadMain() MAC from(tapReadBuf + 6,6); unsigned int etherType = ((((unsigned int)tapReadBuf[12]) & 0xff) << 8) | (((unsigned int)tapReadBuf[13]) & 0xff); try { - // TODO: decode vlans - _handler(_arg,_nwid,from,to,etherType,0,tapReadBuf + 14,bytesRead - 14); + _handler(_arg,(void *)0,_nwid,from,to,etherType,0,tapReadBuf + 14,bytesRead - 14); } catch ( ... ) {} // handlers should not throw } } diff --git a/osdep/WindowsEthernetTap.hpp b/osdep/WindowsEthernetTap.hpp index 53bba3e9..f2cf73f3 100644 --- a/osdep/WindowsEthernetTap.hpp +++ b/osdep/WindowsEthernetTap.hpp @@ -87,7 +87,7 @@ public: unsigned int metric, uint64_t nwid, const char *friendlyName, - void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg); ~WindowsEthernetTap(); @@ -118,7 +118,7 @@ private: void _setRegistryIPv4Value(const char *regKey,const std::vector &value); void _syncIps(); - void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; MAC _mac; uint64_t _nwid; diff --git a/service/OneService.cpp b/service/OneService.cpp index 22eefbb9..c07b3ba4 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -291,21 +291,21 @@ static void _moonToJson(nlohmann::json &mj,const World &world) class OneServiceImpl; -static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf); -static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData); -static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize); -static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure); -static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); -static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); -static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); +static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf); +static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData); +static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize); +static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,const void *data,unsigned long len,int secure); +static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); +static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); +static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); #ifdef ZT_ENABLE_CLUSTER static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len); static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z); #endif -static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); +static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); static int ShttpOnMessageBegin(http_parser *parser); static int ShttpOnUrl(http_parser *parser,const char *ptr,size_t length); @@ -573,7 +573,7 @@ public: cb.eventCallback = SnodeEventCallback; cb.pathCheckFunction = SnodePathCheckFunction; cb.pathLookupFunction = SnodePathLookupFunction; - _node = new Node(this,&cb,OSUtils::now()); + _node = new Node(this,(void *)0,&cb,OSUtils::now()); } // Read local configuration @@ -804,7 +804,7 @@ public: for(std::vector::iterator f(networksDotD.begin());f!=networksDotD.end();++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16)&&(f->substr(16) == ".conf")) - _node->join(Utils::hexStrToU64(f->substr(0,dot).c_str()),(void *)0); + _node->join(Utils::hexStrToU64(f->substr(0,dot).c_str()),(void *)0,(void *)0); } } { // Load existing moons @@ -812,7 +812,7 @@ public: for(std::vector::iterator f(moonsDotD.begin());f!=moonsDotD.end();++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16)&&(f->substr(16) == ".moon")) - _node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str()),0); + _node->orbit((void *)0,Utils::hexStrToU64(f->substr(0,dot).c_str()),0); } } @@ -877,7 +877,7 @@ public: uint64_t dl = _nextBackgroundTaskDeadline; if (dl <= now) { - _node->processBackgroundTasks(now,&_nextBackgroundTaskDeadline); + _node->processBackgroundTasks((void *)0,now,&_nextBackgroundTaskDeadline); dl = _nextBackgroundTaskDeadline; } @@ -892,7 +892,7 @@ public: std::vector added,removed; n->second.tap->scanMulticastGroups(added,removed); for(std::vector::iterator m(added.begin());m!=added.end();++m) - _node->multicastSubscribe(n->first,m->mac().toInt(),m->adi()); + _node->multicastSubscribe((void *)0,n->first,m->mac().toInt(),m->adi()); for(std::vector::iterator m(removed.begin());m!=removed.end();++m) _node->multicastUnsubscribe(n->first,m->mac().toInt(),m->adi()); } @@ -1306,7 +1306,7 @@ public: res["signature"] = json(); res["updatesMustBeSignedBy"] = json(); res["waiting"] = true; - _node->orbit(id,seed); + _node->orbit((void *)0,id,seed); scode = 200; } @@ -1315,7 +1315,7 @@ public: if (ps.size() == 2) { uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); - _node->join(wantnw,(void *)0); // does nothing if we are a member + _node->join(wantnw,(void *)0,(void *)0); // does nothing if we are a member ZT_VirtualNetworkList *nws = _node->networks(); if (nws) { for(unsigned long i=0;inetworkCount;++i) { @@ -1360,7 +1360,7 @@ public: if (ps[0] == "moon") { if (ps.size() == 2) { - _node->deorbit(Utils::hexStrToU64(ps[1].c_str())); + _node->deorbit((void *)0,Utils::hexStrToU64(ps[1].c_str())); res["result"] = true; scode = 200; } // else 404 @@ -1371,7 +1371,7 @@ public: uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str()); for(unsigned long i=0;inetworkCount;++i) { if (nws->networks[i].nwid == wantnw) { - _node->leave(wantnw,(void **)0); + _node->leave(wantnw,(void **)0,(void *)0); res["result"] = true; scode = 200; break; @@ -1693,6 +1693,7 @@ public: _lastDirectReceiveFromGlobal = OSUtils::now(); const ZT_ResultCode rc = _node->processWirePacket( + (void *)0, OSUtils::now(), reinterpret_cast(localAddr), (const struct sockaddr_storage *)from, // Phy<> uses sockaddr_storage, so it'll always be that big @@ -1845,6 +1846,7 @@ public: if (from) { InetAddress fakeTcpLocalInterfaceAddress((uint32_t)0xffffffff,0xffff); const ZT_ResultCode rc = _node->processWirePacket( + (void *)0, OSUtils::now(), reinterpret_cast(&fakeTcpLocalInterfaceAddress), reinterpret_cast(&from), @@ -2255,7 +2257,7 @@ public: inline void tapFrameHandler(uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { - _node->processVirtualNetworkFrame(OSUtils::now(),nwid,from.toInt(),to.toInt(),etherType,vlanId,data,len,&_nextBackgroundTaskDeadline); + _node->processVirtualNetworkFrame((void *)0,OSUtils::now(),nwid,from.toInt(),to.toInt(),etherType,vlanId,data,len,&_nextBackgroundTaskDeadline); } inline void onHttpRequestToServer(TcpConnection *tc) @@ -2426,21 +2428,21 @@ public: } }; -static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf) +static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf) { return reinterpret_cast(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); } -static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData) +static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData) { reinterpret_cast(uptr)->nodeEventCallback(event,metaData); } -static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize) +static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize) { return reinterpret_cast(uptr)->nodeDataStoreGetFunction(name,buf,bufSize,readIndex,totalSize); } -static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure) +static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,const void *data,unsigned long len,int secure) { return reinterpret_cast(uptr)->nodeDataStorePutFunction(name,data,len,secure); } -static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) +static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) { return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); } -static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) +static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); } -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) { return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr,localAddr,remoteAddr); } -static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) +static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) { return reinterpret_cast(uptr)->nodePathLookupFunction(ztaddr,family,result); } #ifdef ZT_ENABLE_CLUSTER @@ -2458,7 +2460,7 @@ static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr, } #endif -static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) +static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); } static int ShttpOnMessageBegin(http_parser *parser) diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 7ecd42b1..7ec377cc 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -175,7 +175,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void std::string lj; lj.push_back((char)VERB_LATEST); lj.append(OSUtils::jsonDump(*latest)); - _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); + _node.sendUserMessage((void *)0,origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); if (_distLog) { fprintf(_distLog,"%.10llx GET_LATEST %u.%u.%u_%u platform %u arch %u vendor %u channel %s -> LATEST %u.%u.%u_%u" ZT_EOL_S,(unsigned long long)origin,rvMaj,rvMin,rvRev,rvBld,rvPlatform,rvArch,rvVendor,rvChannel.c_str(),bestVMaj,bestVMin,bestVRev,bestVBld); fflush(_distLog); @@ -205,7 +205,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append((uint8_t)VERB_GET_DATA); gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } @@ -229,7 +229,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void buf.append(reinterpret_cast(data) + 1,16); buf.append((uint32_t)idx); buf.append(d->second.bin.data() + idx,std::min((unsigned long)ZT_SOFTWARE_UPDATE_CHUNK_SIZE,(unsigned long)(d->second.bin.length() - idx))); - _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); + _node.sendUserMessage((void *)0,origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); //printf(">> DATA @%u\n",(unsigned int)idx); } } @@ -249,7 +249,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append((uint8_t)VERB_GET_DATA); gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } @@ -296,7 +296,7 @@ bool SoftwareUpdater::check(const uint64_t now) ZT_BUILD_ARCHITECTURE, (int)ZT_VENDOR_ZEROTIER, _channel.c_str()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); //printf(">> GET_LATEST\n"); } @@ -343,7 +343,7 @@ bool SoftwareUpdater::check(const uint64_t now) gd.append((uint8_t)VERB_GET_DATA); gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } -- cgit v1.2.3 From 91c9f4cb205169d0ec151293136aa27d113f2090 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 27 Mar 2017 17:33:25 -0700 Subject: Fix TRACE and CLUSTER builds. --- node/Cluster.cpp | 14 +++++++------- node/Node.cpp | 4 ++-- node/Peer.cpp | 4 ++-- node/Switch.cpp | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 52e03ffe..54206f99 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -346,7 +346,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) Mutex::Lock _l(_remotePeers_m); _RemotePeer &rp = _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)]; if (!rp.lastHavePeerReceived) { - RR->topology->saveIdentity(id); + RR->topology->saveIdentity((void *)0,id); RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH); } rp.lastHavePeerReceived = RR->node->now(); @@ -459,7 +459,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) Mutex::Lock _l2(_members[fromMemberId].lock); _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size()); } - RR->sw->send(rendezvousForLocal,true); + RR->sw->send((void *)0,rendezvousForLocal,true); } } } break; @@ -470,7 +470,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) const unsigned int len = dmsg.at(ptr); ptr += 2; Packet outp(rcpt,RR->identity.address(),verb); outp.append(dmsg.field(ptr,len),len); ptr += len; - RR->sw->send(outp,true); + RR->sw->send((void *)0,outp,true); //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); } break; @@ -479,7 +479,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) if (network) { // Copy into a Packet just to conform to Network API. Eventually // will want to refactor. - network->handleConfigChunk(0,Address(),Buffer(dmsg),ptr); + network->handleConfigChunk((void *)0,0,Address(),Buffer(dmsg),ptr); } } break; } @@ -576,7 +576,7 @@ bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { if (i1->ss_family == i2->ss_family) { TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket(*i1,*i2,data,len); + RR->node->putPacket((void *)0,*i1,*i2,data,len); return true; } } @@ -679,7 +679,7 @@ void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPe for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { if (i1->ss_family == i2->ss_family) { TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket(*i1,*i2,data,len); + RR->node->putPacket((void *)0,*i1,*i2,data,len); return; } } @@ -981,7 +981,7 @@ void Cluster::_flush(uint16_t memberId) void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep) { if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) { - Identity queried(RR->topology->getIdentity(Address(remotep.payload(),ZT_ADDRESS_LENGTH))); + Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH))); if (queried) { Buffer<1024> routp; remotep.source().appendTo(routp); diff --git a/node/Node.cpp b/node/Node.cpp index 4e8d6655..e7dc637f 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -285,7 +285,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint #ifdef ZT_ENABLE_CLUSTER // If clustering is enabled we have to call cluster->doPeriodicTasks() very often, so we override normal timer deadline behavior if (RR->cluster) { - RR->sw->doTimerTasks(now); + RR->sw->doTimerTasks(tptr,now); RR->cluster->doPeriodicTasks(); *nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate } else { @@ -686,7 +686,7 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) tmp2[sizeof(tmp2)-1] = (char)0; Utils::snprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); - postEvent(ZT_EVENT_TRACE,tmp1); + postEvent((void *)0,ZT_EVENT_TRACE,tmp1); } #endif // ZT_TRACE diff --git a/node/Peer.cpp b/node/Peer.cpp index 0cc23e33..0795a6ea 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -103,7 +103,7 @@ void Peer::received( } outp.append((uint16_t)redirectTo.port()); outp.armor(_key,true,path->nextOutgoingCounter()); - path->send(RR,outp.data(),outp.size(),now); + path->send(RR,tPtr,outp.data(),outp.size(),now); } else { // For older peers we use RENDEZVOUS to coax them into contacting us elsewhere. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS); @@ -118,7 +118,7 @@ void Peer::received( outp.append(redirectTo.rawIpData(),16); } outp.armor(_key,true,path->nextOutgoingCounter()); - path->send(RR,outp.data(),outp.size(),now); + path->send(RR,tPtr,outp.data(),outp.size(),now); } suboptimalPath = true; } diff --git a/node/Switch.cpp b/node/Switch.cpp index 62674472..56299a9a 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -863,7 +863,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) Packet::Fragment frag(packet,fragStart,chunkSize,fno,totalFragments); #ifdef ZT_ENABLE_CLUSTER if (viaPath) - viaPath->send(RR,frag.data(),frag.size(),now); + viaPath->send(RR,tPtr,frag.data(),frag.size(),now); else if (clusterMostRecentMemberId >= 0) RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,frag.data(),frag.size()); #else -- cgit v1.2.3 From 8a62ba07e57a423c88a503d5162ca205bfd3b529 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 4 Apr 2017 06:47:01 -0700 Subject: Membership cleanup work in progress. --- node/Credential.hpp | 58 +++++++++++ node/Membership.cpp | 280 ++++++++++++---------------------------------------- node/Membership.hpp | 156 ++++++++--------------------- 3 files changed, 162 insertions(+), 332 deletions(-) create mode 100644 node/Credential.hpp (limited to 'node') diff --git a/node/Credential.hpp b/node/Credential.hpp new file mode 100644 index 00000000..0ae2a0a8 --- /dev/null +++ b/node/Credential.hpp @@ -0,0 +1,58 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_CREDENTIAL_HPP +#define ZT_CREDENTIAL_HPP + +#include +#include +#include + +#include +#include +#include +#include + +#include "Constants.hpp" + +namespace ZeroTier { + +/** + * Base class for credentials + */ +class Credential +{ +public: + /** + * Do not change type code IDs -- these are used in Revocation objects and elsewhere + */ + enum Type + { + CREDENTIAL_TYPE_NULL = 0, + CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership + CREDENTIAL_TYPE_CAPABILITY = 2, + CREDENTIAL_TYPE_TAG = 3, + CREDENTIAL_TYPE_COO = 4, // CertificateOfOwnership + CREDENTIAL_TYPE_COR = 5, // CertificateOfRepresentation + CREDENTIAL_TYPE_REVOCATION = 6 + }; +}; + +} // namespace ZeroTier + +#endif diff --git a/node/Membership.cpp b/node/Membership.cpp index 22c13c88..ffe770c6 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -33,11 +33,12 @@ namespace ZeroTier { Membership::Membership() : _lastUpdatedMulticast(0), _lastPushedCom(0), - _comRevocationThreshold(0) + _comRevocationThreshold(0), + _revocations(4), + _remoteTags(4), + _remoteCaps(4), + _remoteCoos(4) { - for(unsigned int i=0;i *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialComp()); - return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); -} - Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) { - const uint64_t newts = com.timestamp().first; + const uint64_t newts = com.timestamp(); if (newts <= _comRevocationThreshold) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (revoked)",com.issuedTo().toString().c_str(),com.networkId()); return ADD_REJECTED; } - const uint64_t oldts = _com.timestamp().first; + const uint64_t oldts = _com.timestamp(); if (newts < oldts) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (older than current)",com.issuedTo().toString().c_str(),com.networkId()); return ADD_REJECTED; @@ -154,84 +149,84 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) +// Template out addCredential() for most cred types to avoid copypasta +template +static Membership::AddCredentialResult _addCredImpl(Hashtable &remoteCreds,const Hashtable &revocations,const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const C &cred) { - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; - if (have) { - if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > tag.timestamp()) ) { - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); - return ADD_REJECTED; + C *rc = remoteCreds.get(cred.id()); + if (rc) { + if (rc->timestamp() >= cred.timestamp()) { + TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (older than credential we have)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_REJECTED; } - if (have->credential == tag) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); - return ADD_ACCEPTED_REDUNDANT; + if (*rc == cred) { + TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (redundant)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_ACCEPTED_REDUNDANT; } } - switch(tag.verify(RR,tPtr)) { + const uint64_t *rt = revocations.get(Membership::revocationKey(C::credentialType(),cred.id())); + if ((rt)&&(*rt >= cred.timestamp())) { + TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (timestamp below revocation threshold)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_REJECTED; + } + + switch(cred.verify(RR,tPtr)) { default: - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); - return ADD_REJECTED; + TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (invalid)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_REJECTED; case 0: - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (!have) have = _newTag(tag.id()); - have->lastReceived = RR->node->now(); - have->credential = tag; - return ADD_ACCEPTED_NEW; + TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (new)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + if (!rc) + rc = &(remoteCreds[cred.id()]); + *rc = cred; + return Membership::ADD_ACCEPTED_NEW; case 1: - return ADD_DEFERRED_FOR_WHOIS; + return Membership::ADD_DEFERRED_FOR_WHOIS; } } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) +{ + return _addCredImpl(_remoteTags,_revocations,RR,tPtr,nconf,tag); +} + Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; - if (have) { - if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > cap.timestamp()) ) { - TRACE("addCredential(Capability) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); - return ADD_REJECTED; - } - if (have->credential == cap) { - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); - return ADD_ACCEPTED_REDUNDANT; - } - } + return _addCredImpl(_remoteCaps,_revocations,RR,tPtr,nconf,cap); +} - switch(cap.verify(RR,tPtr)) { - default: - TRACE("addCredential(Capability) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); - return ADD_REJECTED; - case 0: - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); - if (!have) have = _newCapability(cap.id()); - have->lastReceived = RR->node->now(); - have->credential = cap; - return ADD_ACCEPTED_NEW; - case 1: - return ADD_DEFERRED_FOR_WHOIS; - } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) +{ + return _addCredImpl(_remoteCoos,_revocations,RR,tPtr,nconf,coo); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { + uint64_t *rt; switch(rev.verify(RR,tPtr)) { default: return ADD_REJECTED; case 0: { - const uint64_t now = RR->node->now(); - switch(rev.type()) { + const Credential::Type ct = rev.type(); + switch(ct) { + case Credential::CREDENTIAL_TYPE_COM: + if (rev.threshold() > _comRevocationThreshold) { + _comRevocationThreshold = rev.threshold(); + return ADD_ACCEPTED_NEW; + } + return ADD_ACCEPTED_REDUNDANT; + case Credential::CREDENTIAL_TYPE_CAPABILITY: + case Credential::CREDENTIAL_TYPE_TAG: + case Credential::CREDENTIAL_TYPE_COO: + rt = &(_revocations[revocationKey(ct,rev.credentialId())]); + if (*rt < rev.threshold()) { + *rt = rev.threshold(); + return ADD_ACCEPTED_NEW; + } + return ADD_ACCEPTED_REDUNDANT; default: return ADD_REJECTED; - case Revocation::CREDENTIAL_TYPE_COM: - return (_revokeCom(rev) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); - case Revocation::CREDENTIAL_TYPE_CAPABILITY: - return (_revokeCap(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); - case Revocation::CREDENTIAL_TYPE_TAG: - return (_revokeTag(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); - case Revocation::CREDENTIAL_TYPE_COO: - return (_revokeCoo(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); } } case 1: @@ -239,157 +234,4 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; - if (have) { - if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",coo.issuedTo().toString().c_str(),coo.networkId()); - return ADD_REJECTED; - } - if (have->credential == coo) { - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (redundant)",coo.issuedTo().toString().c_str(),coo.networkId()); - return ADD_ACCEPTED_REDUNDANT; - } - } - - switch(coo.verify(RR,tPtr)) { - default: - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",coo.issuedTo().toString().c_str(),coo.networkId()); - return ADD_REJECTED; - case 0: - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (new)",coo.issuedTo().toString().c_str(),coo.networkId()); - if (!have) have = _newCoo(coo.id()); - have->lastReceived = RR->node->now(); - have->credential = coo; - return ADD_ACCEPTED_NEW; - case 1: - return ADD_DEFERRED_FOR_WHOIS; - } -} - -Membership::_RemoteCredential *Membership::_newTag(const uint64_t id) -{ - _RemoteCredential *t = NULL; - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { - t = _remoteTags[i]; - break; - } else if (_remoteTags[i]->lastReceived <= minlr) { - t = _remoteTags[i]; - minlr = _remoteTags[i]->lastReceived; - } - } - - if (t) { - t->id = id; - t->lastReceived = 0; - t->revocationThreshold = 0; - t->credential = Tag(); - } - - std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialComp()); - return t; -} - -Membership::_RemoteCredential *Membership::_newCapability(const uint64_t id) -{ - _RemoteCredential *c = NULL; - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { - c = _remoteCaps[i]; - break; - } else if (_remoteCaps[i]->lastReceived <= minlr) { - c = _remoteCaps[i]; - minlr = _remoteCaps[i]->lastReceived; - } - } - - if (c) { - c->id = id; - c->lastReceived = 0; - c->revocationThreshold = 0; - c->credential = Capability(); - } - - std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialComp()); - return c; -} - -Membership::_RemoteCredential *Membership::_newCoo(const uint64_t id) -{ - _RemoteCredential *c = NULL; - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { - c = _remoteCoos[i]; - break; - } else if (_remoteCoos[i]->lastReceived <= minlr) { - c = _remoteCoos[i]; - minlr = _remoteCoos[i]->lastReceived; - } - } - - if (c) { - c->id = id; - c->lastReceived = 0; - c->revocationThreshold = 0; - c->credential = CertificateOfOwnership(); - } - - std::sort(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),_RemoteCredentialComp()); - return c; -} - -bool Membership::_revokeCom(const Revocation &rev) -{ - if (rev.threshold() > _comRevocationThreshold) { - _comRevocationThreshold = rev.threshold(); - return true; - } - return false; -} - -bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; - if (!have) have = _newCapability(rev.credentialId()); - if (rev.threshold() > have->revocationThreshold) { - have->lastReceived = now; - have->revocationThreshold = rev.threshold(); - return true; - } - return false; -} - -bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; - if (!have) have = _newTag(rev.credentialId()); - if (rev.threshold() > have->revocationThreshold) { - have->lastReceived = now; - have->revocationThreshold = rev.threshold(); - return true; - } - return false; -} - -bool Membership::_revokeCoo(const Revocation &rev,const uint64_t now) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; - if (!have) have = _newCoo(rev.credentialId()); - if (rev.threshold() > have->revocationThreshold) { - have->lastReceived = now; - have->revocationThreshold = rev.threshold(); - return true; - } - return false; -} - } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index c28d598c..4bd04ad6 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -23,6 +23,8 @@ #include "Constants.hpp" #include "../include/ZeroTierOne.h" +#include "Credential.hpp" +#include "Hashtable.hpp" #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" @@ -49,29 +51,17 @@ private: template struct _RemoteCredential { - _RemoteCredential() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} - uint64_t id; + _RemoteCredential() : lastReceived(0),revocationThreshold(0),credential() {} uint64_t lastReceived; // last time we got this credential uint64_t revocationThreshold; // credentials before this time are invalid T credential; - inline bool operator<(const _RemoteCredential &c) const { return (id < c.id); } }; - template - struct _RemoteCredentialComp - { - inline bool operator()(const _RemoteCredential *a,const _RemoteCredential *b) const { return (a->id < b->id); } - inline bool operator()(const uint64_t a,const _RemoteCredential *b) const { return (a < b->id); } - inline bool operator()(const _RemoteCredential *a,const uint64_t b) const { return (a->id < b); } - inline bool operator()(const uint64_t a,const uint64_t b) const { return (a < b); } - }; - - // Used to track push state for network config tags[] and capabilities[] entries struct _LocalCredentialPushState { _LocalCredentialPushState() : lastPushed(0),id(0) {} uint64_t lastPushed; // last time we sent our own copy of this credential - uint64_t id; + uint32_t id; }; public: @@ -83,72 +73,6 @@ public: ADD_DEFERRED_FOR_WHOIS }; - /** - * Iterator to scan forward through capabilities in ascending order of ID - */ - class CapabilityIterator - { - public: - CapabilityIterator(const Membership &m,const NetworkConfig &nconf) : - _m(&m), - _c(&nconf), - _i(&(m._remoteCaps[0])) {} - - inline const Capability *next() - { - for(;;) { - if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Capability *tmp = &((*_i)->credential); - if (_m->_isCredentialTimestampValid(*_c,**_i)) { - ++_i; - return tmp; - } else ++_i; - } else { - return (const Capability *)0; - } - } - } - - private: - const Membership *_m; - const NetworkConfig *_c; - const _RemoteCredential *const *_i; - }; - friend class CapabilityIterator; - - /** - * Iterator to scan forward through tags in ascending order of ID - */ - class TagIterator - { - public: - TagIterator(const Membership &m,const NetworkConfig &nconf) : - _m(&m), - _c(&nconf), - _i(&(m._remoteTags[0])) {} - - inline const Tag *next() - { - for(;;) { - if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Tag *tmp = &((*_i)->credential); - if (_m->_isCredentialTimestampValid(*_c,**_i)) { - ++_i; - return tmp; - } else ++_i; - } else { - return (const Tag *)0; - } - } - } - - private: - const Membership *_m; - const NetworkConfig *_c; - const _RemoteCredential *const *_i; - }; - friend class TagIterator; - Membership(); /** @@ -190,10 +114,8 @@ public: */ inline bool isAllowedOnNetwork(const NetworkConfig &nconf) const { - if (nconf.isPublic()) - return true; - if (_com.timestamp().first <= _comRevocationThreshold) - return false; + if (nconf.isPublic()) return true; + if (_com.timestamp() <= _comRevocationThreshold) return false; return nconf.com.agreesWith(_com); } @@ -206,23 +128,30 @@ public: * @return True if this peer has a certificate of ownership for the given resource */ template - inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) const + inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) { - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) - break; - if ((_isCredentialTimestampValid(nconf,*_remoteCoos[i]))&&(_remoteCoos[i]->credential.owns(r))) + uint32_t *k = (uint32_t *)0; + CertificateOfOwnership *v = (CertificateOfOwnership *)0; + Hashtable< uint32_t,CertificateOfOwnership >::Iterator i(_remoteCoos); + while (i.next(k,v)) { + if (_isCredentialTimestampValid(nconf,*v)&&(v->owns(r))) return true; } return false; } /** + * Get a remote member's tag (if we have it) + * * @param nconf Network configuration * @param id Tag ID * @return Pointer to tag or NULL if not found */ - const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const; + inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const + { + const Tag *const t = _remoteTags.get(id); + return (((t)&&(_isCredentialTimestampValid(nconf,*t))) ? t : (Tag *)0); + } /** * Validate and add a credential if signature is okay and it's otherwise good @@ -242,29 +171,32 @@ public: /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); -private: - _RemoteCredential *_newTag(const uint64_t id); - _RemoteCredential *_newCapability(const uint64_t id); - _RemoteCredential *_newCoo(const uint64_t id); - bool _revokeCom(const Revocation &rev); - bool _revokeCap(const Revocation &rev,const uint64_t now); - bool _revokeTag(const Revocation &rev,const uint64_t now); - bool _revokeCoo(const Revocation &rev,const uint64_t now); + /** + * Generates a key for the internal revocation tracking hash table + * + * @param t Credential type + * @param i Credential ID + * @return Key for tracking revocations of this credential + */ + static uint64_t revocationKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } +private: template - inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const _RemoteCredential &remoteCredential) const + inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &remoteCredential) const { - if (!remoteCredential.lastReceived) - return false; - const uint64_t ts = remoteCredential.credential.timestamp(); - return ( (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) && (ts > remoteCredential.revocationThreshold) ); + const uint64_t ts = remoteCredential.timestamp(); + if (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) { + const uint64_t *threshold = _revocations.get(revocationKey(C::credentialType(),remoteCredential.id())); + return ((!threshold)||(ts > *threshold)); + } + return false; } // Last time we pushed MULTICAST_LIKE(s) @@ -279,15 +211,13 @@ private: // Remote member's latest network COM CertificateOfMembership _com; - // Sorted (in ascending order of ID) arrays of pointers to remote credentials - _RemoteCredential *_remoteTags[ZT_MAX_NETWORK_TAGS]; - _RemoteCredential *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; - _RemoteCredential *_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Revocations + Hashtable< uint64_t,uint64_t > _revocations; - // This is the RAM allocated for remote credential cache objects - _RemoteCredential _tagMem[ZT_MAX_NETWORK_TAGS]; - _RemoteCredential _capMem[ZT_MAX_NETWORK_CAPABILITIES]; - _RemoteCredential _cooMem[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Remote credentials and credential state + Hashtable< uint32_t,Tag > _remoteTags; + Hashtable< uint32_t,Capability > _remoteCaps; + Hashtable< uint32_t,CertificateOfOwnership > _remoteCoos; // Local credential push state tracking _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; -- cgit v1.2.3 From eddbc7e757f26e59d6eeab7e31e31eb6c47dcf20 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 4 Apr 2017 08:07:38 -0700 Subject: Logic simplification, cleanup, and memory use improvements in Membership. Also fix an issue that may cause network instability in some cases. --- controller/EmbeddedNetworkController.cpp | 2 +- node/Capability.hpp | 5 +- node/CertificateOfMembership.hpp | 25 ++++--- node/CertificateOfOwnership.hpp | 5 +- node/CertificateOfRepresentation.hpp | 6 +- node/Membership.cpp | 48 ++++++------- node/Membership.hpp | 118 ++++++++++++++++++++----------- node/Network.cpp | 30 ++++---- node/Revocation.hpp | 41 +++++------ node/Tag.hpp | 19 ++--- 10 files changed, 175 insertions(+), 124 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 5bce7886..a1a75c9e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -665,7 +665,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( // Member is being de-authorized, so spray Revocation objects to all online members if (!newAuth) { _clearNetworkMemberInfoCache(nwid); - Revocation rev(_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); + Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); Mutex::Lock _l(_lastRequestTime_m); for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { diff --git a/node/Capability.hpp b/node/Capability.hpp index 5ef6c994..454723ac 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -24,6 +24,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Utils.hpp" @@ -58,9 +59,11 @@ class RuntimeEnvironment; * handed off between nodes. Limited transferrability of capabilities is * a feature of true capability based security. */ -class Capability +class Capability : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_CAPABILITY; } + Capability() { memset(this,0,sizeof(Capability)); diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index ae976b50..dfccb138 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -27,6 +27,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "Buffer.hpp" #include "Address.hpp" #include "C25519.hpp" @@ -68,9 +69,11 @@ class RuntimeEnvironment; * This is a memcpy()'able structure and is safe (in a crash sense) to modify * without locks. */ -class CertificateOfMembership +class CertificateOfMembership : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COM; } + /** * Reserved qualifier IDs * @@ -155,18 +158,23 @@ public: /** * @return True if there's something here */ - inline operator bool() const throw() { return (_qualifierCount != 0); } + inline operator bool() const { return (_qualifierCount != 0); } + + /** + * @return Credential ID, always 0 for COMs + */ + inline uint32_t id() const { return 0; } /** * @return Timestamp for this cert and maximum delta for timestamp */ - inline std::pair timestamp() const + inline uint64_t timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) - return std::pair(_qualifiers[i].value,_qualifiers[i].maxDelta); + return _qualifiers[i].value; } - return std::pair(0ULL,0ULL); + return 0; } /** @@ -258,12 +266,12 @@ public: /** * @return True if signed */ - inline bool isSigned() const throw() { return (_signedBy); } + inline bool isSigned() const { return (_signedBy); } /** * @return Address that signed this certificate or null address if none */ - inline const Address &signedBy() const throw() { return _signedBy; } + inline const Address &signedBy() const { return _signedBy; } template inline void serialize(Buffer &b) const @@ -321,7 +329,6 @@ public: } inline bool operator==(const CertificateOfMembership &c) const - throw() { if (_signedBy != c._signedBy) return false; @@ -335,7 +342,7 @@ public: } return (_signature == c._signature); } - inline bool operator!=(const CertificateOfMembership &c) const throw() { return (!(*this == c)); } + inline bool operator!=(const CertificateOfMembership &c) const { return (!(*this == c)); } private: struct _Qualifier diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 8c47582d..93be64dd 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -25,6 +25,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "C25519.hpp" #include "Address.hpp" #include "Identity.hpp" @@ -45,9 +46,11 @@ class RuntimeEnvironment; /** * Certificate indicating ownership of a network identifier */ -class CertificateOfOwnership +class CertificateOfOwnership : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COO; } + enum Thing { THING_NULL = 0, diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp index 02e961c4..710ee577 100644 --- a/node/CertificateOfRepresentation.hpp +++ b/node/CertificateOfRepresentation.hpp @@ -20,6 +20,7 @@ #define ZT_CERTIFICATEOFREPRESENTATION_HPP #include "Constants.hpp" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Identity.hpp" @@ -47,14 +48,17 @@ namespace ZeroTier { * roots can shield nodes entirely and p2p connectivity behind them can * be disabled. This will be desirable for a number of use cases. */ -class CertificateOfRepresentation +class CertificateOfRepresentation : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COR; } + CertificateOfRepresentation() { memset(this,0,sizeof(CertificateOfRepresentation)); } + inline uint32_t id() const { return 0; } inline uint64_t timestamp() const { return _timestamp; } inline const Address &representative(const unsigned int i) const { return _reps[i]; } inline unsigned int repCount() const { return _repCount; } diff --git a/node/Membership.cpp b/node/Membership.cpp index ffe770c6..62c07314 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -39,6 +39,7 @@ Membership::Membership() : _remoteCaps(4), _remoteCoos(4) { + resetPushState(); } void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) @@ -48,18 +49,16 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u const Capability *sendCap; if (localCapabilityIndex >= 0) { sendCap = &(nconf.capabilities[localCapabilityIndex]); - if ( (_localCaps[localCapabilityIndex].id != sendCap->id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCaps[localCapabilityIndex].lastPushed = now; - _localCaps[localCapabilityIndex].id = sendCap->id(); - } else sendCap = (const Capability *)0; + if ( ((now - _localCredLastPushed.cap[localCapabilityIndex]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) + _localCredLastPushed.cap[localCapabilityIndex] = now; + else sendCap = (const Capability *)0; } else sendCap = (const Capability *)0; const Tag *sendTags[ZT_MAX_NETWORK_TAGS]; unsigned int sendTagCount = 0; for(unsigned int t=0;t= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localTags[t].lastPushed = now; - _localTags[t].id = nconf.tags[t].id(); + if ( ((now - _localCredLastPushed.tag[t]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCredLastPushed.tag[t] = now; sendTags[sendTagCount++] = &(nconf.tags[t]); } } @@ -67,9 +66,8 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u const CertificateOfOwnership *sendCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; unsigned int sendCooCount = 0; for(unsigned int c=0;c= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCoos[c].lastPushed = now; - _localCoos[c].id = nconf.certificatesOfOwnership[c].id(); + if ( ((now - _localCredLastPushed.coo[c]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCredLastPushed.coo[c] = now; sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); } } @@ -149,7 +147,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -// Template out addCredential() for most cred types to avoid copypasta +// Template out addCredential() for many cred types to avoid copypasta template static Membership::AddCredentialResult _addCredImpl(Hashtable &remoteCreds,const Hashtable &revocations,const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const C &cred) { @@ -165,7 +163,7 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot } } - const uint64_t *rt = revocations.get(Membership::revocationKey(C::credentialType(),cred.id())); + const uint64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); if ((rt)&&(*rt >= cred.timestamp())) { TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (timestamp below revocation threshold)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); return Membership::ADD_REJECTED; @@ -186,20 +184,9 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) -{ - return _addCredImpl(_remoteTags,_revocations,RR,tPtr,nconf,tag); -} - -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) -{ - return _addCredImpl(_remoteCaps,_revocations,RR,tPtr,nconf,cap); -} - -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) -{ - return _addCredImpl(_remoteCoos,_revocations,RR,tPtr,nconf,coo); -} +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) { return _addCredImpl(_remoteTags,_revocations,RR,tPtr,nconf,tag); } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { return _addCredImpl(_remoteCaps,_revocations,RR,tPtr,nconf,cap); } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { return _addCredImpl(_remoteCoos,_revocations,RR,tPtr,nconf,coo); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { @@ -219,7 +206,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme case Credential::CREDENTIAL_TYPE_CAPABILITY: case Credential::CREDENTIAL_TYPE_TAG: case Credential::CREDENTIAL_TYPE_COO: - rt = &(_revocations[revocationKey(ct,rev.credentialId())]); + rt = &(_revocations[credentialKey(ct,rev.credentialId())]); if (*rt < rev.threshold()) { *rt = rev.threshold(); return ADD_ACCEPTED_NEW; @@ -234,4 +221,11 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } +void Membership::clean(const uint64_t now,const NetworkConfig &nconf) +{ + _cleanCredImpl(nconf,_remoteTags); + _cleanCredImpl(nconf,_remoteCaps); + _cleanCredImpl(nconf,_remoteCoos); +} + } // namespace ZeroTier diff --git a/node/Membership.hpp b/node/Membership.hpp index 4bd04ad6..22772859 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -47,23 +47,6 @@ class Network; */ class Membership { -private: - template - struct _RemoteCredential - { - _RemoteCredential() : lastReceived(0),revocationThreshold(0),credential() {} - uint64_t lastReceived; // last time we got this credential - uint64_t revocationThreshold; // credentials before this time are invalid - T credential; - }; - - struct _LocalCredentialPushState - { - _LocalCredentialPushState() : lastPushed(0),id(0) {} - uint64_t lastPushed; // last time we sent our own copy of this credential - uint32_t id; - }; - public: enum AddCredentialResult { @@ -92,19 +75,19 @@ public: void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** - * Check whether we should push MULTICAST_LIKEs to this peer + * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true * * @param now Current time * @return True if we should update multicasts */ - inline bool shouldLikeMulticasts(const uint64_t now) const { return ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD); } - - /** - * Set time we last updated multicasts for this peer - * - * @param now Current time - */ - inline void likingMulticasts(const uint64_t now) { _lastUpdatedMulticast = now; } + inline bool multicastLikeGate(const uint64_t now) + { + if ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD) { + _lastUpdatedMulticast = now; + return true; + } + return false; + } /** * Check whether the peer represented by this Membership should be allowed on this network at all @@ -128,11 +111,11 @@ public: * @return True if this peer has a certificate of ownership for the given resource */ template - inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) + inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) const { uint32_t *k = (uint32_t *)0; CertificateOfOwnership *v = (CertificateOfOwnership *)0; - Hashtable< uint32_t,CertificateOfOwnership >::Iterator i(_remoteCoos); + Hashtable< uint32_t,CertificateOfOwnership >::Iterator i(*(const_cast< Hashtable< uint32_t,CertificateOfOwnership> *>(&_remoteCoos))); while (i.next(k,v)) { if (_isCredentialTimestampValid(nconf,*v)&&(v->owns(r))) return true; @@ -179,13 +162,28 @@ public: AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); /** - * Generates a key for the internal revocation tracking hash table + * Clean internal databases of stale entries + * + * @param now Current time + * @param nconf Current network configuration + */ + void clean(const uint64_t now,const NetworkConfig &nconf); + + /** + * Reset last pushed time for local credentials * - * @param t Credential type - * @param i Credential ID - * @return Key for tracking revocations of this credential + * This is done when we update our network configuration and our credentials have changed + */ + inline void resetPushState() + { + _lastPushedCom = 0; + memset(&_localCredLastPushed,0,sizeof(_localCredLastPushed)); + } + + /** + * Generates a key for the internal use in indexing credentials by type and credential ID */ - static uint64_t revocationKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } + static uint64_t credentialKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } private: template @@ -193,12 +191,24 @@ private: { const uint64_t ts = remoteCredential.timestamp(); if (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) { - const uint64_t *threshold = _revocations.get(revocationKey(C::credentialType(),remoteCredential.id())); + const uint64_t *threshold = _revocations.get(credentialKey(C::credentialType(),remoteCredential.id())); return ((!threshold)||(ts > *threshold)); } return false; } + template + void _cleanCredImpl(const NetworkConfig &nconf,Hashtable &remoteCreds) + { + uint32_t *k = (uint32_t *)0; + C *v = (C *)0; + typename Hashtable::Iterator i(remoteCreds); + while (i.next(k,v)) { + if (!_isCredentialTimestampValid(nconf,*v)) + remoteCreds.erase(*k); + } + } + // Last time we pushed MULTICAST_LIKE(s) uint64_t _lastUpdatedMulticast; @@ -211,18 +221,46 @@ private: // Remote member's latest network COM CertificateOfMembership _com; - // Revocations + // Revocations by credentialKey() Hashtable< uint64_t,uint64_t > _revocations; - // Remote credentials and credential state + // Remote credentials that we have received from this member (and that are valid) Hashtable< uint32_t,Tag > _remoteTags; Hashtable< uint32_t,Capability > _remoteCaps; Hashtable< uint32_t,CertificateOfOwnership > _remoteCoos; - // Local credential push state tracking - _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; - _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; - _LocalCredentialPushState _localCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Time we last pushed our local credentials to this member + struct { + uint64_t tag[ZT_MAX_NETWORK_TAGS]; + uint64_t cap[ZT_MAX_NETWORK_CAPABILITIES]; + uint64_t coo[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + } _localCredLastPushed; + +public: + class CapabilityIterator + { + public: + CapabilityIterator(Membership &m,const NetworkConfig &nconf) : + _hti(m._remoteCaps), + _k((uint32_t *)0), + _c((Capability *)0), + _nconf(nconf) + { + } + + inline Capability *next() + { + if (_hti.next(_k,_c)) + return _c; + else return (Capability *)0; + } + + private: + Hashtable< uint32_t,Capability >::Iterator _hti; + uint32_t *_k; + Capability *_c; + const NetworkConfig &_nconf; + }; }; } // namespace ZeroTier diff --git a/node/Network.cpp b/node/Network.cpp index 0abfdf86..3c607b28 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -534,9 +534,9 @@ static _doZtFilterResult _doZtFilter( } if (inbound) { if (membership) { - if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) + if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED; - if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) + if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_MAC_AUTHENTICATED; } } else { @@ -1143,21 +1143,31 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD // _lock is NOT locked when this is called try { if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) - return 0; + return 0; // invalid config that is not for us or not for this network if (_config == nconf) return 1; // OK config, but duplicate of what we already have ZT_VirtualNetworkConfig ctmp; bool oldPortInitialized; - { + { // do things that require lock here, but unlock before calling callbacks Mutex::Lock _l(_lock); + _config = nconf; _lastConfigUpdate = RR->node->now(); _netconfFailure = NETCONF_FAILURE_NONE; + oldPortInitialized = _portInitialized; _portInitialized = true; + _externalConfig(&ctmp); + + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) + m->resetPushState(); } + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { @@ -1299,10 +1309,9 @@ bool Network::gate(void *tPtr,const SharedPtr &peer) if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { if (!m) m = &(_membership(peer->address())); - if (m->shouldLikeMulticasts(now)) { + if (m->multicastLikeGate(now)) { m->pushCredentials(RR,tPtr,now,peer->address(),_config,-1,false); _announceMulticastGroupsTo(tPtr,peer->address(),_allMulticastGroups()); - m->likingMulticasts(now); } return true; } @@ -1338,6 +1347,7 @@ void Network::clean() while (i.next(a,m)) { if (!RR->topology->getPeerNoCache(*a)) _memberships.erase(*a); + else m->clean(now,_config); } } } @@ -1546,8 +1556,7 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu } // Make sure that all "network anchors" have Membership records so we will - // push multicasts to them. Note that _membership() also does this but in a - // piecemeal on-demand fashion. + // push multicasts to them. const std::vector
anchors(_config.anchors()); for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) _membership(*a); @@ -1559,11 +1568,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu Hashtable::Iterator i(_memberships); while (i.next(a,m)) { m->pushCredentials(RR,tPtr,now,*a,_config,-1,false); - if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { - if (!newMulticastGroup) - m->likingMulticasts(now); + if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config)) ) _announceMulticastGroupsTo(tPtr,*a,groups); - } } } } diff --git a/node/Revocation.hpp b/node/Revocation.hpp index 8b9ce6dd..e5e013bd 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -26,6 +26,7 @@ #include "Constants.hpp" #include "../include/ZeroTierOne.h" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Utils.hpp" @@ -44,20 +45,10 @@ class RuntimeEnvironment; /** * Revocation certificate to instantaneously revoke a COM, capability, or tag */ -class Revocation +class Revocation : public Credential { public: - /** - * Credential type being revoked - */ - enum CredentialType - { - CREDENTIAL_TYPE_NULL = 0, - CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership - CREDENTIAL_TYPE_CAPABILITY = 2, - CREDENTIAL_TYPE_TAG = 3, - CREDENTIAL_TYPE_COO = 4 // CertificateOfOwnership - }; + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_REVOCATION; } Revocation() { @@ -73,23 +64,23 @@ public: * @param tgt Target node whose credential(s) are being revoked * @param ct Credential type being revoked */ - Revocation(const uint64_t i,const uint64_t nwid,const uint64_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const CredentialType ct) : + Revocation(const uint32_t i,const uint64_t nwid,const uint32_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const Credential::Type ct) : _id(i), - _networkId(nwid), _credentialId(cid), + _networkId(nwid), _threshold(thr), _flags(fl), _target(tgt), _signedBy(), _type(ct) {} - inline uint64_t id() const { return _id; } + inline uint32_t id() const { return _id; } + inline uint32_t credentialId() const { return _credentialId; } inline uint64_t networkId() const { return _networkId; } - inline uint64_t credentialId() const { return _credentialId; } inline uint64_t threshold() const { return _threshold; } inline const Address &target() const { return _target; } inline const Address &signer() const { return _signedBy; } - inline CredentialType type() const { return _type; } + inline Credential::Type type() const { return _type; } inline bool fastPropagate() const { return ((_flags & ZT_REVOCATION_FLAG_FAST_PROPAGATE) != 0); } @@ -123,8 +114,10 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + b.append((uint32_t)0); // 4 unused bytes, currently set to 0 b.append(_id); b.append(_networkId); + b.append((uint32_t)0); // 4 unused bytes, currently set to 0 b.append(_credentialId); b.append(_threshold); b.append(_flags); @@ -151,14 +144,16 @@ public: unsigned int p = startAt; - _id = b.template at(p); p += 8; + p += 4; // 4 bytes, currently unused + _id = b.template at(p); p += 4; _networkId = b.template at(p); p += 8; - _credentialId = b.template at(p); p += 8; + p += 4; // 4 bytes, currently unused + _credentialId = b.template at(p); p += 4; _threshold = b.template at(p); p += 8; _flags = b.template at(p); p += 8; _target.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - _type = (CredentialType)b[p++]; + _type = (Credential::Type)b[p++]; if (b[p++] == 1) { if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { @@ -178,14 +173,14 @@ public: } private: - uint64_t _id; + uint32_t _id; + uint32_t _credentialId; uint64_t _networkId; - uint64_t _credentialId; uint64_t _threshold; uint64_t _flags; Address _target; Address _signedBy; - CredentialType _type; + Credential::Type _type; C25519::Signature _signature; }; diff --git a/node/Tag.hpp b/node/Tag.hpp index 38085906..1f7f6835 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -25,6 +25,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "C25519.hpp" #include "Address.hpp" #include "Identity.hpp" @@ -51,9 +52,11 @@ class RuntimeEnvironment; * Unlike capabilities tags are signed only by the issuer and are never * transferrable. */ -class Tag +class Tag : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_TAG; } + Tag() { memset(this,0,sizeof(Tag)); @@ -67,19 +70,19 @@ public: * @param value Tag value */ Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : - _networkId(nwid), - _ts(ts), _id(id), _value(value), + _networkId(nwid), + _ts(ts), _issuedTo(issuedTo), _signedBy() { } - inline uint64_t networkId() const { return _networkId; } - inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } + inline uint64_t networkId() const { return _networkId; } + inline uint64_t timestamp() const { return _ts; } inline const Address &issuedTo() const { return _issuedTo; } inline const Address &signedBy() const { return _signedBy; } @@ -115,11 +118,9 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - // These are the same between Tag and Capability b.append(_networkId); b.append(_ts); b.append(_id); - b.append(_value); _issuedTo.appendTo(b); @@ -187,10 +188,10 @@ public: }; private: - uint64_t _networkId; - uint64_t _ts; uint32_t _id; uint32_t _value; + uint64_t _networkId; + uint64_t _ts; Address _issuedTo; Address _signedBy; C25519::Signature _signature; -- cgit v1.2.3 From 5ad120208f4d9864952b2ce8b3e62293421e9c10 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 4 Apr 2017 08:46:12 -0700 Subject: Small fix, should filter by temporal validity. --- node/Membership.hpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Membership.hpp b/node/Membership.hpp index 22772859..0bc8f335 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -244,21 +244,25 @@ public: _hti(m._remoteCaps), _k((uint32_t *)0), _c((Capability *)0), + _m(m), _nconf(nconf) { } inline Capability *next() { - if (_hti.next(_k,_c)) - return _c; - else return (Capability *)0; + while (_hti.next(_k,_c)) { + if (_m._isCredentialTimestampValid(_nconf,*_c)) + return _c; + } + return (Capability *)0; } private: Hashtable< uint32_t,Capability >::Iterator _hti; uint32_t *_k; Capability *_c; + Membership &_m; const NetworkConfig &_nconf; }; }; -- cgit v1.2.3 From 88a4a3b1bae97548142b73031ff415db6ebd31d0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 11 Apr 2017 08:47:02 -0700 Subject: Pass tptr on leave. --- node/Network.cpp | 12 +++++++++++- node/Network.hpp | 7 +++---- node/Node.cpp | 21 +++++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) (limited to 'node') diff --git a/node/Network.cpp b/node/Network.cpp index 3c607b28..b7f25f7f 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -729,7 +729,8 @@ Network::~Network() char n[128]; if (_destroyed) { - RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + // This is done in Node::leave() so we can pass tPtr + //RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); RR->node->dataStoreDelete((void *)0,n); } else { @@ -993,6 +994,9 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) { + if (_destroyed) + return 0; + const unsigned int start = ptr; ptr += 8; // skip network ID, which is already obviously known @@ -1140,6 +1144,9 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToDisk) { + if (_destroyed) + return 0; + // _lock is NOT locked when this is called try { if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) @@ -1190,6 +1197,9 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD void Network::requestConfiguration(void *tPtr) { + if (_destroyed) + return; + /* ZeroTier addresses can't begin with 0xff, so this is used to mark controllerless * network IDs. Controllerless network IDs only support unicast IPv6 using the 6plane * addressing scheme and have the following format: 0xffSSSSEEEE000000 where SSSS diff --git a/node/Network.hpp b/node/Network.hpp index fccc267a..faef0fed 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -344,9 +344,8 @@ public: /** * Destroy this network * - * This causes the network to disable itself, destroy its tap device, and on - * delete to delete all trace of itself on disk and remove any persistent tap - * device instances. Call this when a network is being removed from the system. + * This sets the network to completely remove itself on delete. This also prevents the + * call of the normal port shutdown event on delete. */ void destroy(); @@ -364,7 +363,7 @@ public: /** * @return Externally usable pointer-to-pointer exported via the core API */ - inline void **userPtr() throw() { return &_uPtr; } + inline void **userPtr() { return &_uPtr; } private: ZT_VirtualNetworkStatus _status() const; diff --git a/node/Node.cpp b/node/Node.cpp index e7dc637f..9844b09e 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -305,26 +305,35 @@ ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr) { Mutex::Lock _l(_networks_m); SharedPtr nw = _network(nwid); - if(!nw) - _networks.push_back(std::pair< uint64_t,SharedPtr >(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr)))); - std::sort(_networks.begin(),_networks.end()); // will sort by nwid since it's the first in a pair<> + if(!nw) { + const std::pair< uint64_t,SharedPtr > nn(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr))); + _networks.insert(std::upper_bound(_networks.begin(),_networks.end(),nn),nn); + } return ZT_RESULT_OK; } ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { + ZT_VirtualNetworkConfig ctmp; std::vector< std::pair< uint64_t,SharedPtr > > newn; + void **nUserPtr = (void **)0; Mutex::Lock _l(_networks_m); + for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { - if (n->first != nwid) + if (n->first != nwid) { newn.push_back(*n); - else { + } else { if (uptr) - *uptr = n->second->userPtr(); + *uptr = *n->second->userPtr(); n->second->destroy(); + nUserPtr = n->second->userPtr(); } } _networks.swap(newn); + + if (nUserPtr) + RR->node->configureVirtualNetworkPort(tptr,nwid,nUserPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + return ZT_RESULT_OK; } -- cgit v1.2.3 From 139c4b56337c0cfe7458ecf5df4e12e38c2d4f8a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 14 Apr 2017 17:53:32 -0700 Subject: Significant simplification to path logic. --- node/Constants.hpp | 2 +- node/IncomingPacket.cpp | 32 +++--- node/Node.cpp | 18 ++-- node/Peer.cpp | 273 +++++++++++++++++++----------------------------- node/Peer.hpp | 119 +++++++++------------ node/Topology.hpp | 4 +- 6 files changed, 182 insertions(+), 266 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 410a245b..93184efa 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -289,7 +289,7 @@ #define ZT_PEER_PING_PERIOD 60000 /** - * Paths are considered expired if they have not produced a real packet in this long + * Paths are considered expired if they have not sent us a real packet in this long */ #define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 52794fd7..a0f5ee1d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1200,16 +1200,12 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt switch(addrType) { case 4: { - InetAddress a(field(ptr,4),4,at(ptr + 4)); - - bool redundant = false; - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->setClusterOptimal(a); - } else { - redundant = peer->hasActivePathTo(now,a); - } - - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { + const InetAddress a(field(ptr,4),4,at(ptr + 4)); + if ( + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path + { if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); @@ -1219,16 +1215,12 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt } } break; case 6: { - InetAddress a(field(ptr,16),16,at(ptr + 16)); - - bool redundant = false; - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->setClusterOptimal(a); - } else { - redundant = peer->hasActivePathTo(now,a); - } - - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { + const InetAddress a(field(ptr,16),16,at(ptr + 16)); + if ( + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path + { if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); diff --git a/node/Node.cpp b/node/Node.cpp index 9844b09e..1bc96cca 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -407,17 +407,17 @@ ZT_PeerList *Node::peers() const p->latency = pi->second->latency(); p->role = RR->topology->role(pi->second->identity().address()); - std::vector< std::pair< SharedPtr,bool > > paths(pi->second->paths(_now)); + std::vector< SharedPtr > paths(pi->second->paths(_now)); SharedPtr bestp(pi->second->getBestPath(_now,false)); p->pathCount = 0; - for(std::vector< std::pair< SharedPtr,bool > >::iterator path(paths.begin());path!=paths.end();++path) { - memcpy(&(p->paths[p->pathCount].address),&(path->first->address()),sizeof(struct sockaddr_storage)); - p->paths[p->pathCount].lastSend = path->first->lastOut(); - p->paths[p->pathCount].lastReceive = path->first->lastIn(); - p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->first->address()); - p->paths[p->pathCount].linkQuality = (int)path->first->linkQuality(); - p->paths[p->pathCount].expired = path->second; - p->paths[p->pathCount].preferred = (path->first == bestp) ? 1 : 0; + for(std::vector< SharedPtr >::iterator path(paths.begin());path!=paths.end();++path) { + memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); + p->paths[p->pathCount].lastSend = (*path)->lastOut(); + p->paths[p->pathCount].lastReceive = (*path)->lastIn(); + p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); + p->paths[p->pathCount].linkQuality = (int)(*path)->linkQuality(); + p->paths[p->pathCount].expired = 0; + p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0; ++p->pathCount; } } diff --git a/node/Peer.cpp b/node/Peer.cpp index 0795a6ea..2711dd19 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -27,14 +27,6 @@ #include "Cluster.hpp" #include "Packet.hpp" -#ifndef AF_MAX -#if AF_INET > AF_INET6 -#define AF_MAX AF_INET -#else -#define AF_MAX AF_INET6 -#endif -#endif - namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : @@ -51,18 +43,15 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastComRequestSent(0), _lastCredentialsReceived(0), _lastTrustEstablishedPacketReceived(0), - _remoteClusterOptimal4(0), _vProto(0), _vMajor(0), _vMinor(0), _vRevision(0), _id(peerIdentity), - _numPaths(0), _latency(0), _directPathPushCutoffCount(0), _credentialsCutoffCount(0) { - memset(_remoteClusterOptimal6,0,sizeof(_remoteClusterOptimal6)); if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw std::runtime_error("new peer identity key agreement failed"); } @@ -80,7 +69,7 @@ void Peer::received( const uint64_t now = RR->node->now(); #ifdef ZT_ENABLE_CLUSTER - bool suboptimalPath = false; + bool isClusterSuboptimalPath = false; if ((RR->cluster)&&(hops == 0)) { // Note: findBetterEndpoint() is first since we still want to check // for a better endpoint even if we don't actually send a redirect. @@ -146,65 +135,60 @@ void Peer::received( path->updateLinkQuality((unsigned int)(packetId & 7)); if (hops == 0) { - bool pathIsConfirmed = false; + bool pathAlreadyKnown = false; { Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address() == path->address()) { - _paths[p].lastReceive = now; - _paths[p].path = path; // local address may have changed! + if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) { + const struct sockaddr_in *const r = reinterpret_cast(&(path->address())); + const struct sockaddr_in *const l = reinterpret_cast(&(_v4Path.p->address())); + const struct sockaddr_in *const rl = reinterpret_cast(&(path->localAddress())); + const struct sockaddr_in *const ll = reinterpret_cast(&(_v4Path.p->localAddress())); + if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(rl->sin_addr.s_addr == ll->sin_addr.s_addr)&&(rl->sin_port == ll->sin_port)) { + _v4Path.lr = now; #ifdef ZT_ENABLE_CLUSTER - _paths[p].localClusterSuboptimal = suboptimalPath; + _v4Path.localClusterSuboptimal = isClusterSuboptimalPath; #endif - pathIsConfirmed = true; - break; + pathAlreadyKnown = true; + } + } else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) { + const struct sockaddr_in6 *const r = reinterpret_cast(&(path->address())); + const struct sockaddr_in6 *const l = reinterpret_cast(&(_v6Path.p->address())); + const struct sockaddr_in6 *const rl = reinterpret_cast(&(path->localAddress())); + const struct sockaddr_in6 *const ll = reinterpret_cast(&(_v6Path.p->localAddress())); + if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(!memcmp(rl->sin6_addr.s6_addr,ll->sin6_addr.s6_addr,16))&&(rl->sin6_port == ll->sin6_port)) { + _v6Path.lr = now; +#ifdef ZT_ENABLE_CLUSTER + _v6Path.localClusterSuboptimal = isClusterSuboptimalPath; +#endif + pathAlreadyKnown = true; } } } - if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { + if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); - - // Since this is a new path, figure out where to put it (possibly replacing an old/dead one) - unsigned int slot; - if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) { - slot = _numPaths++; - } else { - // First try to replace the worst within the same address family, if possible - int worstSlot = -1; - uint64_t worstScore = 0xffffffffffffffffULL; - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address().ss_family == path->address().ss_family) { - const uint64_t s = _pathScore(p,now); - if (s < worstScore) { - worstScore = s; - worstSlot = (int)p; - } - } - } - if (worstSlot >= 0) { - slot = (unsigned int)worstSlot; - } else { - // If we can't find one with the same family, replace the worst of any family - slot = ZT_MAX_PEER_NETWORK_PATHS - 1; - for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _pathScore(p,now); - if (s < worstScore) { - worstScore = s; - slot = p; - } - } + if (path->address().ss_family == AF_INET) { + if ((!_v4Path.p)||(!_v4Path.p->alive(now))||(path->preferenceRank() >= _v4Path.p->preferenceRank())) { + _v4Path.lr = now; + _v4Path.p = path; +#ifdef ZT_ENABLE_CLUSTER + _v4Path.localClusterSuboptimal = isClusterSuboptimalPath; + if (RR->cluster) + RR->cluster->broadcastHavePeer(_id); +#endif } - } - - _paths[slot].lastReceive = now; - _paths[slot].path = path; + } else if (path->address().ss_family == AF_INET6) { + if ((!_v6Path.p)||(!_v6Path.p->alive(now))||(path->preferenceRank() >= _v6Path.p->preferenceRank())) { + _v6Path.lr = now; + _v6Path.p = path; #ifdef ZT_ENABLE_CLUSTER - _paths[slot].localClusterSuboptimal = suboptimalPath; - if (RR->cluster) - RR->cluster->broadcastHavePeer(_id); + _v6Path.localClusterSuboptimal = isClusterSuboptimalPath; + if (RR->cluster) + RR->cluster->broadcastHavePeer(_id); #endif + } + } } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); @@ -214,10 +198,10 @@ void Peer::received( } else if (this->trustEstablished(now)) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) #ifdef ZT_ENABLE_CLUSTER - // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection - const bool haveCluster = (RR->cluster); + // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection + const bool haveCluster = (RR->cluster); #else - const bool haveCluster = false; + const bool haveCluster = false; #endif if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) { _lastDirectPathPushSent = now; @@ -290,60 +274,50 @@ void Peer::received( } } -bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const -{ - Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if ( (_paths[p].path->address() == addr) && ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)) ) - return true; - } - return false; -} - -bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) +bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) { Mutex::Lock _l(_paths_m); - int bestp = -1; - uint64_t best = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)||(forceEvenIfDead)) ) { - const uint64_t s = _pathScore(p,now); - if (s >= best) { - best = s; - bestp = (int)p; - } + uint64_t v6lr = 0; + if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) + v6lr = _v6Path.p->lastIn(); + uint64_t v4lr = 0; + if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) + v4lr = _v4Path.p->lastIn(); + + if ( (v6lr > v4lr) && ((now - v6lr) < ZT_PATH_ALIVE_TIMEOUT) ) { + return _v6Path.p->send(RR,tPtr,data,len,now); + } else if ((now - v4lr) < ZT_PATH_ALIVE_TIMEOUT) { + return _v4Path.p->send(RR,tPtr,data,len,now); + } else if (force) { + if (v6lr > v4lr) { + return _v6Path.p->send(RR,tPtr,data,len,now); + } else if (v4lr) { + return _v4Path.p->send(RR,tPtr,data,len,now); } } - if (bestp >= 0) { - return _paths[bestp].path->send(RR,tPtr,data,len,now); - } else { - return false; - } + return false; } SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) { Mutex::Lock _l(_paths_m); - int bestp = -1; - uint64_t best = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) || (includeExpired) ) { - const uint64_t s = _pathScore(p,now); - if (s >= best) { - best = s; - bestp = (int)p; - } - } + uint64_t v6lr = 0; + if ( ( includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v6Path.p) ) + v6lr = _v6Path.p->lastIn(); + uint64_t v4lr = 0; + if ( ( includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v4Path.p) ) + v4lr = _v4Path.p->lastIn(); + + if (v6lr > v4lr) { + return _v6Path.p; + } else if (v4lr) { + return _v4Path.p; } - if (bestp >= 0) { - return _paths[bestp].path; - } else { - return SharedPtr(); - } + return SharedPtr(); } void Peer::sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) @@ -420,79 +394,44 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); - int bestp = -1; - uint64_t best = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && ((inetAddressFamily < 0)||((int)_paths[p].path->address().ss_family == inetAddressFamily)) ) { - const uint64_t s = _pathScore(p,now); - if (s >= best) { - best = s; - bestp = (int)p; + if (inetAddressFamily < 0) { + uint64_t v6lr = 0; + if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) + v6lr = _v6Path.p->lastIn(); + uint64_t v4lr = 0; + if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) + v4lr = _v4Path.p->lastIn(); + + if (v6lr > v4lr) { + if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + _v6Path.p->sent(now); + return true; + } + } else if (v4lr) { + if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + _v4Path.p->sent(now); + return true; } } - } - - if (bestp >= 0) { - if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); - _paths[bestp].path->sent(now); - } - return true; } else { - return false; - } -} - -bool Peer::hasActiveDirectPath(uint64_t now) const -{ - Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if (((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION)&&(_paths[p].path->alive(now))) - return true; - } - return false; -} - -void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) -{ - Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - attemptToContactAt(tPtr,_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); - _paths[p].path->sent(now); - _paths[p].lastReceive = 0; // path will not be used unless it speaks again - } - } -} - -void Peer::getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const -{ - Mutex::Lock _l(_paths_m); - - int bestp4 = -1,bestp6 = -1; - uint64_t best4 = 0ULL,best6 = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)) ) { - if (_paths[p].path->address().ss_family == AF_INET) { - const uint64_t s = _pathScore(p,now); - if (s >= best4) { - best4 = s; - bestp4 = (int)p; - } - } else if (_paths[p].path->address().ss_family == AF_INET6) { - const uint64_t s = _pathScore(p,now); - if (s >= best6) { - best6 = s; - bestp6 = (int)p; - } + if ( (inetAddressFamily == AF_INET) && ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { + if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + _v4Path.p->sent(now); + return true; + } + } else if ( (inetAddressFamily == AF_INET6) && ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { + if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + _v6Path.p->sent(now); + return true; } } } - if (bestp4 >= 0) - v4 = _paths[bestp4].path->address(); - if (bestp6 >= 0) - v6 = _paths[bestp6].path->address(); + return false; } } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 41836410..f225eb85 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -108,20 +108,10 @@ public: * @param addr Remote address * @return True if we have an active path to this destination */ - bool hasActivePathTo(uint64_t now,const InetAddress &addr) const; - - /** - * Set which known path for an address family is optimal - * - * @param addr Address to make exclusive - */ - inline void setClusterOptimal(const InetAddress &addr) + inline bool hasActivePathTo(uint64_t now,const InetAddress &addr) const { - if (addr.ss_family == AF_INET) { - _remoteClusterOptimal4 = (uint32_t)reinterpret_cast(&addr)->sin_addr.s_addr; - } else if (addr.ss_family == AF_INET6) { - memcpy(_remoteClusterOptimal6,reinterpret_cast(&addr)->sin6_addr.s6_addr,16); - } + Mutex::Lock _l(_paths_m); + return ( ((addr.ss_family == AF_INET)&&(_v4Path.p)&&(_v4Path.p->address() == addr)&&(_v4Path.p->alive(now))) || ((addr.ss_family == AF_INET6)&&(_v6Path.p)&&(_v6Path.p->address() == addr)&&(_v6Path.p->alive(now))) ); } /** @@ -131,14 +121,17 @@ public: * @param data Packet data * @param len Packet length * @param now Current time - * @param forceEvenIfDead If true, send even if the path is not 'alive' + * @param force If true, send even if path is not alive * @return True if we actually sent something */ - bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); + bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force); /** * Get the best current direct path * + * This does not check Path::alive(), but does return the most recently + * active path and does check expiration (which is a longer timeout). + * * @param now Current time * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none @@ -192,12 +185,6 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); - /** - * @param now Current time - * @return True if this peer has at least one active and alive direct path - */ - bool hasActiveDirectPath(uint64_t now) const; - /** * Reset paths within a given IP scope and address family * @@ -209,30 +196,48 @@ public: * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); + inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) + { + Mutex::Lock _l(_paths_m); + if ((inetAddressFamily == AF_INET)&&(_v4Path.lr)&&(_v4Path.p->address().ipScope() == scope)) { + attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + _v4Path.p->sent(now); + _v4Path.lr = 0; // path will not be used unless it speaks again + } else if ((inetAddressFamily == AF_INET6)&&(_v6Path.lr)&&(_v6Path.p->address().ipScope() == scope)) { + attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + _v6Path.p->sent(now); + _v6Path.lr = 0; // path will not be used unless it speaks again + } + } /** - * Get most recently active path addresses for IPv4 and/or IPv6 - * - * Note that v4 and v6 are not modified if they are not found, so - * initialize these to a NULL address to be able to check. + * Fill parameters with V4 and V6 addresses if known and alive * * @param now Current time * @param v4 Result parameter to receive active IPv4 address, if any * @param v6 Result parameter to receive active IPv6 address, if any */ - void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; + inline void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const + { + Mutex::Lock _l(_paths_m); + if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) + v4 = _v4Path.p->address(); + if (((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v6Path.p->alive(now))) + v6 = _v6Path.p->address(); + } /** * @param now Current time - * @return All known direct paths to this peer and whether they are expired (true == expired) + * @return All known paths to this peer */ - inline std::vector< std::pair< SharedPtr,bool > > paths(const uint64_t now) const + inline std::vector< SharedPtr > paths(const uint64_t now) const { - std::vector< std::pair< SharedPtr,bool > > pp; + std::vector< SharedPtr > pp; Mutex::Lock _l(_paths_m); - for(unsigned int p=0,np=_numPaths;p,bool >(_paths[p].path,(now - _paths[p].lastReceive) > ZT_PEER_PATH_EXPIRATION)); + if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) + pp.push_back(_v4Path.p); + if (((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v6Path.p->alive(now))) + pp.push_back(_v6Path.p); return pp; } @@ -424,32 +429,19 @@ public: } private: - inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const + struct _PeerPath { - uint64_t s = ZT_PEER_PING_PERIOD + _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); - - if (_paths[p].path->address().ss_family == AF_INET) { - s += (uint64_t)(ZT_PEER_PING_PERIOD * (unsigned long)(reinterpret_cast(&(_paths[p].path->address()))->sin_addr.s_addr == _remoteClusterOptimal4)); - } else if (_paths[p].path->address().ss_family == AF_INET6) { - uint64_t clusterWeight = ZT_PEER_PING_PERIOD; - const uint8_t *a = reinterpret_cast(reinterpret_cast(&(_paths[p].path->address()))->sin6_addr.s6_addr); - for(long i=0;i<16;++i) { - if (a[i] != _remoteClusterOptimal6[i]) { - clusterWeight = 0; - break; - } - } - s += clusterWeight; - } - - s += (ZT_PEER_PING_PERIOD / 2) * (uint64_t)_paths[p].path->alive(now); - #ifdef ZT_ENABLE_CLUSTER - s -= ZT_PEER_PING_PERIOD * (uint64_t)_paths[p].localClusterSuboptimal; + _PeerPath() : lr(0),p(),localClusterSuboptimal(false) {} +#else + _PeerPath() : lr(0),p() {} #endif - - return s; - } + uint64_t lr; // time of last valid ZeroTier packet + SharedPtr p; +#ifdef ZT_ENABLE_CLUSTER + bool localClusterSuboptimal; // true if our cluster has determined that we should not be serving this peer +#endif + }; uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; @@ -468,26 +460,17 @@ private: uint64_t _lastCredentialsReceived; uint64_t _lastTrustEstablishedPacketReceived; - uint8_t _remoteClusterOptimal6[16]; - uint32_t _remoteClusterOptimal4; - uint16_t _vProto; uint16_t _vMajor; uint16_t _vMinor; uint16_t _vRevision; - Identity _id; - - struct { - uint64_t lastReceive; - SharedPtr path; -#ifdef ZT_ENABLE_CLUSTER - bool localClusterSuboptimal; -#endif - } _paths[ZT_MAX_PEER_NETWORK_PATHS]; + _PeerPath _v4Path; // IPv4 direct path + _PeerPath _v6Path; // IPv6 direct path Mutex _paths_m; - unsigned int _numPaths; + Identity _id; + unsigned int _latency; unsigned int _directPathPushCutoffCount; unsigned int _credentialsCutoffCount; diff --git a/node/Topology.hpp b/node/Topology.hpp index 4870ab5e..d29c424e 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -314,7 +314,9 @@ public: Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - cnt += (unsigned long)((*p)->hasActiveDirectPath(now)); + const SharedPtr pp((*p)->getBestPath(now,false)); + if ((pp)&&(pp->alive(now))) + ++cnt; } return cnt; } -- cgit v1.2.3 From f1c0563c40dc9e3ec5e975d3e1e8d6057ed6bd83 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 14 Apr 2017 18:02:04 -0700 Subject: Fix for cluster handoff. --- node/IncomingPacket.cpp | 4 ++++ node/Peer.cpp | 4 ++-- node/Peer.hpp | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index a0f5ee1d..303160ec 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1206,6 +1206,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) + peer->setClusterPreferred(a); if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); @@ -1221,6 +1223,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) + peer->setClusterPreferred(a); if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); diff --git a/node/Peer.cpp b/node/Peer.cpp index 2711dd19..7ffe8926 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -169,7 +169,7 @@ void Peer::received( if (verb == Packet::VERB_OK) { Mutex::Lock _l(_paths_m); if (path->address().ss_family == AF_INET) { - if ((!_v4Path.p)||(!_v4Path.p->alive(now))||(path->preferenceRank() >= _v4Path.p->preferenceRank())) { + if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) { _v4Path.lr = now; _v4Path.p = path; #ifdef ZT_ENABLE_CLUSTER @@ -179,7 +179,7 @@ void Peer::received( #endif } } else if (path->address().ss_family == AF_INET6) { - if ((!_v6Path.p)||(!_v6Path.p->alive(now))||(path->preferenceRank() >= _v6Path.p->preferenceRank())) { + if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) { _v6Path.lr = now; _v6Path.p = path; #ifdef ZT_ENABLE_CLUSTER diff --git a/node/Peer.hpp b/node/Peer.hpp index f225eb85..6cf30feb 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -210,6 +210,19 @@ public: } } + /** + * Indicate that the given address was provided by a cluster as a preferred destination + * + * @param addr Address cluster prefers that we use + */ + inline void setClusterPreferred(const InetAddress &addr) + { + if (addr.ss_family == AF_INET) + _v4ClusterPreferred = addr; + else if (addr.ss_family == AF_INET6) + _v6ClusterPreferred = addr; + } + /** * Fill parameters with V4 and V6 addresses if known and alive * @@ -465,6 +478,9 @@ private: uint16_t _vMinor; uint16_t _vRevision; + InetAddress _v4ClusterPreferred; + InetAddress _v6ClusterPreferred; + _PeerPath _v4Path; // IPv4 direct path _PeerPath _v6Path; // IPv6 direct path Mutex _paths_m; -- cgit v1.2.3 From 2487a8bede10f1ea58ca2778d9772ee9056e6b12 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 09:14:21 -0700 Subject: Fix for 100% cpu issue. --- node/Peer.cpp | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index 7ffe8926..827dc7de 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -166,33 +166,31 @@ void Peer::received( } if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { - if (verb == Packet::VERB_OK) { - Mutex::Lock _l(_paths_m); - if (path->address().ss_family == AF_INET) { - if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) { - _v4Path.lr = now; - _v4Path.p = path; -#ifdef ZT_ENABLE_CLUSTER - _v4Path.localClusterSuboptimal = isClusterSuboptimalPath; - if (RR->cluster) - RR->cluster->broadcastHavePeer(_id); -#endif - } - } else if (path->address().ss_family == AF_INET6) { - if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) { - _v6Path.lr = now; - _v6Path.p = path; + Mutex::Lock _l(_paths_m); + _PeerPath *potentialNewPeerPath = (_PeerPath *)0; + if (path->address().ss_family == AF_INET) { + if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) { + potentialNewPeerPath = &_v4Path; + } + } else if (path->address().ss_family == AF_INET6) { + if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) { + potentialNewPeerPath = &_v6Path; + } + } + if (potentialNewPeerPath) { + if (verb == Packet::VERB_OK) { + potentialNewPeerPath->lr = now; + potentialNewPeerPath->p = path; #ifdef ZT_ENABLE_CLUSTER - _v6Path.localClusterSuboptimal = isClusterSuboptimalPath; - if (RR->cluster) - RR->cluster->broadcastHavePeer(_id); + potentialNewPeerPath->localClusterSuboptimal = isClusterSuboptimalPath; + if (RR->cluster) + RR->cluster->broadcastHavePeer(_id); #endif - } + } else { + TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); + attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); + path->sent(now); } - } else { - TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); - path->sent(now); } } } else if (this->trustEstablished(now)) { -- cgit v1.2.3 From 1d8ded3293495ceb98553a0b446311e86bb00ee2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 09:30:28 -0700 Subject: Tiny largely non-consequential credential fix. --- node/Membership.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index 62c07314..ceba24b2 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -153,7 +153,7 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot { C *rc = remoteCreds.get(cred.id()); if (rc) { - if (rc->timestamp() >= cred.timestamp()) { + if (rc->timestamp() > cred.timestamp()) { TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (older than credential we have)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); return Membership::ADD_REJECTED; } -- cgit v1.2.3 From ba5d0cc2f91310bb5e395c241a21aad7e5ab88fa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 09:31:07 -0700 Subject: Silence some TRACE noise. --- node/Membership.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Membership.cpp b/node/Membership.cpp index ceba24b2..2d0471f1 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -158,7 +158,7 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot return Membership::ADD_REJECTED; } if (*rc == cred) { - TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (redundant)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + //TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (redundant)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); return Membership::ADD_ACCEPTED_REDUNDANT; } } -- cgit v1.2.3 From 95e5345cc37316c264c8d3d732d324c60f18ab72 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 10:12:13 -0700 Subject: Cluster build fix. --- node/Peer.cpp | 2 +- node/Peer.hpp | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index 827dc7de..2e9f6a2b 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -109,7 +109,7 @@ void Peer::received( outp.armor(_key,true,path->nextOutgoingCounter()); path->send(RR,tPtr,outp.data(),outp.size(),now); } - suboptimalPath = true; + isClusterSuboptimalPath = true; } } #endif diff --git a/node/Peer.hpp b/node/Peer.hpp index 6cf30feb..b9d85404 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -316,11 +316,8 @@ public: */ inline bool hasLocalClusterOptimalPath(uint64_t now) const { - for(unsigned int p=0,np=_numPaths;palive(now)) && (!_paths[p].localClusterSuboptimal) ) - return true; - } - return false; + Mutex::Lock _l(_paths_m); + return ( ((_v4Path.p)&&(_v4Path.p->alive(now))&&(!_v4Path.localClusterSuboptimal)) || ((_v6Path.p)&&(_v6Path.p->alive(now))&&(!_v6Path.localClusterSuboptimal)) ); } #endif -- cgit v1.2.3 From d8f5cfdee4665451960505d375bd7a20fb0d6f04 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 15:45:49 -0700 Subject: Windows profile build target (CPU profiling), and a little bit of optimization revealed by such. --- node/Poly1305.cpp | 21 ++++--- windows/ZeroTierOne.sln | 31 +++++++++++ windows/ZeroTierOne/ZeroTierOne.vcxproj | 74 ++++++++++++++++++++++++- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 3 + 4 files changed, 121 insertions(+), 8 deletions(-) (limited to 'node') diff --git a/node/Poly1305.cpp b/node/Poly1305.cpp index b78071f6..13d4712d 100644 --- a/node/Poly1305.cpp +++ b/node/Poly1305.cpp @@ -135,11 +135,12 @@ typedef struct poly1305_context { unsigned char opaque[136]; } poly1305_context; -#if (defined(_MSC_VER) || defined(__GNUC__)) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__)) +#if (defined(_MSC_VER) || defined(__GNUC__)) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64)) ////////////////////////////////////////////////////////////////////////////// // 128-bit implementation for MSC and GCC from Poly1305-donna + #if defined(_MSC_VER) #include @@ -183,9 +184,9 @@ typedef struct poly1305_state_internal_t { unsigned char final; } poly1305_state_internal_t; -/* interpret eight 8 bit unsigned integers as a 64 bit unsigned integer in little endian */ -static inline unsigned long long -U8TO64(const unsigned char *p) { +#if defined(ZT_NO_TYPE_PUNNING) || (__BYTE_ORDER != __LITTLE_ENDIAN) +static inline unsigned long long U8TO64(const unsigned char *p) +{ return (((unsigned long long)(p[0] & 0xff) ) | ((unsigned long long)(p[1] & 0xff) << 8) | @@ -196,10 +197,13 @@ U8TO64(const unsigned char *p) { ((unsigned long long)(p[6] & 0xff) << 48) | ((unsigned long long)(p[7] & 0xff) << 56)); } +#else +#define U8TO64(p) (*reinterpret_cast(p)) +#endif -/* store a 64 bit unsigned integer as eight 8 bit unsigned integers in little endian */ -static inline void -U64TO8(unsigned char *p, unsigned long long v) { +#if defined(ZT_NO_TYPE_PUNNING) || (__BYTE_ORDER != __LITTLE_ENDIAN) +static inline void U64TO8(unsigned char *p, unsigned long long v) +{ p[0] = (v ) & 0xff; p[1] = (v >> 8) & 0xff; p[2] = (v >> 16) & 0xff; @@ -209,6 +213,9 @@ U64TO8(unsigned char *p, unsigned long long v) { p[6] = (v >> 48) & 0xff; p[7] = (v >> 56) & 0xff; } +#else +#define U64TO8(p,v) ((*reinterpret_cast(p)) = (v)) +#endif static inline void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) { diff --git a/windows/ZeroTierOne.sln b/windows/ZeroTierOne.sln index 9baadd81..05f498fb 100644 --- a/windows/ZeroTierOne.sln +++ b/windows/ZeroTierOne.sln @@ -28,6 +28,10 @@ Global DVD-5|Mixed Platforms = DVD-5|Mixed Platforms DVD-5|Win32 = DVD-5|Win32 DVD-5|x64 = DVD-5|x64 + Profile|Any CPU = Profile|Any CPU + Profile|Mixed Platforms = Profile|Mixed Platforms + Profile|Win32 = Profile|Win32 + Profile|x64 = Profile|x64 Release|Any CPU = Release|Any CPU Release|Mixed Platforms = Release|Mixed Platforms Release|Win32 = Release|Win32 @@ -91,6 +95,13 @@ Global {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.DVD-5|x64.ActiveCfg = Debug|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.DVD-5|x64.Build.0 = Debug|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.DVD-5|x64.Deploy.0 = Debug|x64 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|Any CPU.ActiveCfg = Profile|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|Mixed Platforms.ActiveCfg = Profile|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|Mixed Platforms.Build.0 = Profile|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|Win32.ActiveCfg = Profile|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|Win32.Build.0 = Profile|Win32 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|x64.ActiveCfg = Profile|x64 + {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Profile|x64.Build.0 = Profile|x64 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Release|Any CPU.ActiveCfg = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Release|Mixed Platforms.ActiveCfg = Release|Win32 {B00A4957-5977-4AC1-9EF4-571DC27EADA2}.Release|Mixed Platforms.Build.0 = Release|Win32 @@ -189,6 +200,10 @@ Global {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.DVD-5|x64.ActiveCfg = Win8 Release|x64 {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.DVD-5|x64.Build.0 = Win8 Release|x64 {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.DVD-5|x64.Deploy.0 = Win8 Release|x64 + {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Profile|Any CPU.ActiveCfg = Win8 Debug|x64 + {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Profile|Mixed Platforms.ActiveCfg = Win8 Debug|Win32 + {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Profile|Win32.ActiveCfg = Win8 Debug|Win32 + {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Profile|x64.ActiveCfg = Win8 Debug|x64 {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Release|Any CPU.ActiveCfg = Win8 Release|Win32 {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Release|Mixed Platforms.ActiveCfg = Win8 Release|Win32 {43BA7584-D4DB-4F7C-90FC-E2B18A68A213}.Release|Mixed Platforms.Build.0 = Win8 Release|Win32 @@ -287,6 +302,14 @@ Global {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.DVD-5|Mixed Platforms.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.DVD-5|Win32.ActiveCfg = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.DVD-5|x64.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|Any CPU.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|Any CPU.Build.0 = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|Mixed Platforms.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|Mixed Platforms.Build.0 = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|Win32.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|Win32.Build.0 = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|x64.ActiveCfg = Debug|Any CPU + {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Profile|x64.Build.0 = Debug|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Release|Any CPU.ActiveCfg = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Release|Any CPU.Build.0 = Release|Any CPU {4CCA6B98-5E64-45BF-AC34-19B3E2570DB1}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU @@ -361,6 +384,14 @@ Global {6D27214A-087B-4484-B898-AD2A13FA3B9E}.DVD-5|Win32.Build.0 = Debug|Any CPU {6D27214A-087B-4484-B898-AD2A13FA3B9E}.DVD-5|x64.ActiveCfg = Debug|Any CPU {6D27214A-087B-4484-B898-AD2A13FA3B9E}.DVD-5|x64.Build.0 = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|Any CPU.ActiveCfg = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|Any CPU.Build.0 = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|Mixed Platforms.ActiveCfg = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|Mixed Platforms.Build.0 = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|Win32.ActiveCfg = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|Win32.Build.0 = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|x64.ActiveCfg = Debug|Any CPU + {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Profile|x64.Build.0 = Debug|Any CPU {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Release|Any CPU.ActiveCfg = Release|Any CPU {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Release|Any CPU.Build.0 = Release|Any CPU {6D27214A-087B-4484-B898-AD2A13FA3B9E}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 32d7fa4a..84a44198 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -9,6 +9,14 @@ Debug x64 + + Profile + Win32 + + + Profile + x64 + Release Win32 @@ -66,6 +74,7 @@ false + false @@ -113,6 +122,7 @@ + @@ -172,12 +182,24 @@ v140 MultiByte + + Application + true + v140 + MultiByte + Application true v140 MultiByte + + Application + true + v140 + MultiByte + Application false @@ -198,9 +220,15 @@ + + + + + + @@ -213,6 +241,11 @@ $(SolutionDir)\Build\$(Platform)\$(Configuration)\ zerotier-one_x86 + + .exe + $(SolutionDir)\Build\$(Platform)\$(Configuration)\ + zerotier-one_x86 + .exe $(SolutionDir)\Build\$(Platform)\$(Configuration)\ @@ -223,6 +256,11 @@ $(SolutionDir)\Build\$(Platform)\$(Configuration)\ zerotier-one_x64 + + .exe + $(SolutionDir)\Build\$(Platform)\$(Configuration)\ + zerotier-one_x64 + .exe $(SolutionDir)\Build\$(Platform)\$(Configuration)\ @@ -244,6 +282,22 @@ false + + + Level3 + Disabled + true + + + NOMINMAX;STATICLIB;WIN32;ZT_TRACE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions) + 4996 + + + true + wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;%(AdditionalDependencies) + false + + Level3 @@ -262,6 +316,24 @@ "notelemetry.obj" %(AdditionalOptions) + + + Level3 + Disabled + true + + + NOMINMAX;STATICLIB;WIN32;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="disable";%(PreprocessorDefinitions) + false + 4996 + + + true + wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;%(AdditionalDependencies) + false + "notelemetry.obj" %(AdditionalOptions) + + Level3 @@ -291,7 +363,7 @@ Level3 - MaxSpeed + Full true true true diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index ca1640e9..b051b4f7 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -482,6 +482,9 @@ Header Files\node + + Header Files\node + -- cgit v1.2.3 From df48738ac96a6eab5e3baa03f6dd1fb62bdc8040 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 16:43:03 -0700 Subject: Enable use of NaCl for faster X64 Salsa20 implementations. Also include binary for OSX for easy build. Blazingly fast. --- ext/bin/cnacl-osx-amd64/README.md | 53 ++++++++++++ .../include/sodium/crypto_auth_hmacsha256.h | 27 ++++++ .../include/sodium/crypto_auth_hmacsha512256.h | 27 ++++++ .../sodium/crypto_box_curve25519xsalsa20poly1305.h | 44 ++++++++++ .../include/sodium/crypto_core_hsalsa20.h | 27 ++++++ .../include/sodium/crypto_core_salsa20.h | 27 ++++++ .../include/sodium/crypto_core_salsa2012.h | 27 ++++++ .../include/sodium/crypto_core_salsa208.h | 27 ++++++ .../include/sodium/crypto_hash_sha256.h | 22 +++++ .../include/sodium/crypto_hash_sha512.h | 22 +++++ .../include/sodium/crypto_hashblocks_sha256.h | 23 ++++++ .../include/sodium/crypto_hashblocks_sha512.h | 23 ++++++ .../include/sodium/crypto_onetimeauth_poly1305.h | 27 ++++++ .../include/sodium/crypto_scalarmult_curve25519.h | 27 ++++++ .../sodium/crypto_secretbox_xsalsa20poly1305.h | 31 +++++++ .../sodium/crypto_sign_edwards25519sha512batch.h | 32 ++++++++ .../include/sodium/crypto_stream_aes128ctr.h | 35 ++++++++ .../include/sodium/crypto_stream_salsa20.h | 34 ++++++++ .../include/sodium/crypto_stream_salsa2012.h | 34 ++++++++ .../include/sodium/crypto_stream_salsa208.h | 34 ++++++++ .../include/sodium/crypto_stream_xsalsa20.h | 34 ++++++++ .../cnacl-osx-amd64/include/sodium/crypto_types.h | 11 +++ .../include/sodium/crypto_verify_16.h | 21 +++++ .../include/sodium/crypto_verify_32.h | 21 +++++ make-mac.mk | 6 ++ node/Identity.cpp | 2 +- node/Node.cpp | 6 +- node/Packet.cpp | 6 +- node/Salsa20.cpp | 29 +++---- node/Salsa20.hpp | 95 ++++++++++++++++++---- node/Utils.cpp | 2 +- selftest.cpp | 12 +-- 32 files changed, 801 insertions(+), 47 deletions(-) create mode 100644 ext/bin/cnacl-osx-amd64/README.md create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h (limited to 'node') diff --git a/ext/bin/cnacl-osx-amd64/README.md b/ext/bin/cnacl-osx-amd64/README.md new file mode 100644 index 00000000..35426286 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/README.md @@ -0,0 +1,53 @@ +# cNaCl + + If you would like to be confusing, you could pronounce it sea-salt + +This is a fork NaCl by Daniel J. Bernstein and Tanja Lange. +The build has been ported to cmake so it can be cross compiled and build output is reliable. +Since it uses cmake, it could theoretically be built on windows but this has not been tested. +It does compile using mingw32. + +## How do I make this thing work? + + mkdir cbuild + cd cbuild + cmake .. + make + +## Ok now how about cross compiling? + + mkdir cbuildw32 + cd cbuildw32 + cmake -DCMAKE_TOOLCHAIN_FILE=../CMakeWindows.txt .. + make + +## Why fork? + +NaCl builds using a shell script called `./do`. This script does compiling, testing, measuring +and selection of the best implementation of each algorithm for the given machine. It also generates +the header files which will be used. + +The problems with `./do` are it's slow, it tries compiling with multiple different compiler +profiles, it's very platform independent but it doesn't run on Windows and most importantly, with +compiling, testing and measuring so tightly bound, it is impossible to cross compile for a +different operating system. + + +## How it works + +The first time you build for a new ABI, it will trigger the traditional nacl `./do` script. +What cNaCl does is parse the resulting headers from the `./do` build and create a plan so that it +can repeat roughly the same build. + +If there is already a plan for the given ABI, the build uses this plan and the build is very fast. + +Plans are stored in `./cmake/plans/` and I will be adding plans as I find new ones. + + +## What else is new? + +There is a problem with the `./do` build which prevents it from running on some ARM based machines, +this was fixed by adding a more lax method for measuring CPU speed as a fall back. + + +`#EOF#` diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h new file mode 100644 index 00000000..6b5600f3 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h @@ -0,0 +1,27 @@ +#ifndef crypto_auth_hmacsha256_H +#define crypto_auth_hmacsha256_H + +#define crypto_auth_hmacsha256_ref_BYTES 32 +#define crypto_auth_hmacsha256_ref_KEYBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_auth_hmacsha256_ref(const std::string &,const std::string &); +extern void crypto_auth_hmacsha256_ref_verify(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_auth_hmacsha256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_auth_hmacsha256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_auth_hmacsha256 crypto_auth_hmacsha256_ref +#define crypto_auth_hmacsha256_verify crypto_auth_hmacsha256_ref_verify +#define crypto_auth_hmacsha256_BYTES crypto_auth_hmacsha256_ref_BYTES +#define crypto_auth_hmacsha256_KEYBYTES crypto_auth_hmacsha256_ref_KEYBYTES +#define crypto_auth_hmacsha256_IMPLEMENTATION "crypto_auth/hmacsha256/ref" +#ifndef crypto_auth_hmacsha256_ref_VERSION +#define crypto_auth_hmacsha256_ref_VERSION "-" +#endif +#define crypto_auth_hmacsha256_VERSION crypto_auth_hmacsha256_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h new file mode 100644 index 00000000..c9bd96e4 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h @@ -0,0 +1,27 @@ +#ifndef crypto_auth_hmacsha512256_H +#define crypto_auth_hmacsha512256_H + +#define crypto_auth_hmacsha512256_ref_BYTES 32 +#define crypto_auth_hmacsha512256_ref_KEYBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_auth_hmacsha512256_ref(const std::string &,const std::string &); +extern void crypto_auth_hmacsha512256_ref_verify(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_auth_hmacsha512256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_auth_hmacsha512256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_auth_hmacsha512256 crypto_auth_hmacsha512256_ref +#define crypto_auth_hmacsha512256_verify crypto_auth_hmacsha512256_ref_verify +#define crypto_auth_hmacsha512256_BYTES crypto_auth_hmacsha512256_ref_BYTES +#define crypto_auth_hmacsha512256_KEYBYTES crypto_auth_hmacsha512256_ref_KEYBYTES +#define crypto_auth_hmacsha512256_IMPLEMENTATION "crypto_auth/hmacsha512256/ref" +#ifndef crypto_auth_hmacsha512256_ref_VERSION +#define crypto_auth_hmacsha512256_ref_VERSION "-" +#endif +#define crypto_auth_hmacsha512256_VERSION crypto_auth_hmacsha512256_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h new file mode 100644 index 00000000..e2c3b4cc --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h @@ -0,0 +1,44 @@ +#ifndef crypto_box_curve25519xsalsa20poly1305_H +#define crypto_box_curve25519xsalsa20poly1305_H + +#define crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES 24 +#define crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES 16 +#ifdef __cplusplus +#include +extern std::string crypto_box_curve25519xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &,const std::string &); +extern std::string crypto_box_curve25519xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &,const std::string &); +extern std::string crypto_box_curve25519xsalsa20poly1305_ref_keypair(std::string *); +extern "C" { +#endif +extern int crypto_box_curve25519xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_keypair(unsigned char *,unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_beforenm(unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_open_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_box_curve25519xsalsa20poly1305 crypto_box_curve25519xsalsa20poly1305_ref +#define crypto_box_curve25519xsalsa20poly1305_open crypto_box_curve25519xsalsa20poly1305_ref_open +#define crypto_box_curve25519xsalsa20poly1305_keypair crypto_box_curve25519xsalsa20poly1305_ref_keypair +#define crypto_box_curve25519xsalsa20poly1305_beforenm crypto_box_curve25519xsalsa20poly1305_ref_beforenm +#define crypto_box_curve25519xsalsa20poly1305_afternm crypto_box_curve25519xsalsa20poly1305_ref_afternm +#define crypto_box_curve25519xsalsa20poly1305_open_afternm crypto_box_curve25519xsalsa20poly1305_ref_open_afternm +#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES +#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES +#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES +#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES +#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES +#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES +#define crypto_box_curve25519xsalsa20poly1305_IMPLEMENTATION "crypto_box/curve25519xsalsa20poly1305/ref" +#ifndef crypto_box_curve25519xsalsa20poly1305_ref_VERSION +#define crypto_box_curve25519xsalsa20poly1305_ref_VERSION "-" +#endif +#define crypto_box_curve25519xsalsa20poly1305_VERSION crypto_box_curve25519xsalsa20poly1305_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h new file mode 100644 index 00000000..abae188e --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_hsalsa20_H +#define crypto_core_hsalsa20_H + +#define crypto_core_hsalsa20_ref_OUTPUTBYTES 32 +#define crypto_core_hsalsa20_ref_INPUTBYTES 16 +#define crypto_core_hsalsa20_ref_KEYBYTES 32 +#define crypto_core_hsalsa20_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_hsalsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_hsalsa20 crypto_core_hsalsa20_ref +#define crypto_core_hsalsa20_OUTPUTBYTES crypto_core_hsalsa20_ref_OUTPUTBYTES +#define crypto_core_hsalsa20_INPUTBYTES crypto_core_hsalsa20_ref_INPUTBYTES +#define crypto_core_hsalsa20_KEYBYTES crypto_core_hsalsa20_ref_KEYBYTES +#define crypto_core_hsalsa20_CONSTBYTES crypto_core_hsalsa20_ref_CONSTBYTES +#define crypto_core_hsalsa20_IMPLEMENTATION "crypto_core/hsalsa20/ref" +#ifndef crypto_core_hsalsa20_ref_VERSION +#define crypto_core_hsalsa20_ref_VERSION "-" +#endif +#define crypto_core_hsalsa20_VERSION crypto_core_hsalsa20_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h new file mode 100644 index 00000000..9737b101 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_salsa20_H +#define crypto_core_salsa20_H + +#define crypto_core_salsa20_ref_OUTPUTBYTES 64 +#define crypto_core_salsa20_ref_INPUTBYTES 16 +#define crypto_core_salsa20_ref_KEYBYTES 32 +#define crypto_core_salsa20_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_salsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_salsa20 crypto_core_salsa20_ref +#define crypto_core_salsa20_OUTPUTBYTES crypto_core_salsa20_ref_OUTPUTBYTES +#define crypto_core_salsa20_INPUTBYTES crypto_core_salsa20_ref_INPUTBYTES +#define crypto_core_salsa20_KEYBYTES crypto_core_salsa20_ref_KEYBYTES +#define crypto_core_salsa20_CONSTBYTES crypto_core_salsa20_ref_CONSTBYTES +#define crypto_core_salsa20_IMPLEMENTATION "crypto_core/salsa20/ref" +#ifndef crypto_core_salsa20_ref_VERSION +#define crypto_core_salsa20_ref_VERSION "-" +#endif +#define crypto_core_salsa20_VERSION crypto_core_salsa20_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h new file mode 100644 index 00000000..137cd47f --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_salsa2012_H +#define crypto_core_salsa2012_H + +#define crypto_core_salsa2012_ref_OUTPUTBYTES 64 +#define crypto_core_salsa2012_ref_INPUTBYTES 16 +#define crypto_core_salsa2012_ref_KEYBYTES 32 +#define crypto_core_salsa2012_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_salsa2012_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_salsa2012 crypto_core_salsa2012_ref +#define crypto_core_salsa2012_OUTPUTBYTES crypto_core_salsa2012_ref_OUTPUTBYTES +#define crypto_core_salsa2012_INPUTBYTES crypto_core_salsa2012_ref_INPUTBYTES +#define crypto_core_salsa2012_KEYBYTES crypto_core_salsa2012_ref_KEYBYTES +#define crypto_core_salsa2012_CONSTBYTES crypto_core_salsa2012_ref_CONSTBYTES +#define crypto_core_salsa2012_IMPLEMENTATION "crypto_core/salsa2012/ref" +#ifndef crypto_core_salsa2012_ref_VERSION +#define crypto_core_salsa2012_ref_VERSION "-" +#endif +#define crypto_core_salsa2012_VERSION crypto_core_salsa2012_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h new file mode 100644 index 00000000..4895bbbe --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_salsa208_H +#define crypto_core_salsa208_H + +#define crypto_core_salsa208_ref_OUTPUTBYTES 64 +#define crypto_core_salsa208_ref_INPUTBYTES 16 +#define crypto_core_salsa208_ref_KEYBYTES 32 +#define crypto_core_salsa208_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_salsa208_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_salsa208 crypto_core_salsa208_ref +#define crypto_core_salsa208_OUTPUTBYTES crypto_core_salsa208_ref_OUTPUTBYTES +#define crypto_core_salsa208_INPUTBYTES crypto_core_salsa208_ref_INPUTBYTES +#define crypto_core_salsa208_KEYBYTES crypto_core_salsa208_ref_KEYBYTES +#define crypto_core_salsa208_CONSTBYTES crypto_core_salsa208_ref_CONSTBYTES +#define crypto_core_salsa208_IMPLEMENTATION "crypto_core/salsa208/ref" +#ifndef crypto_core_salsa208_ref_VERSION +#define crypto_core_salsa208_ref_VERSION "-" +#endif +#define crypto_core_salsa208_VERSION crypto_core_salsa208_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h new file mode 100644 index 00000000..20d18703 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h @@ -0,0 +1,22 @@ +#ifndef crypto_hash_sha256_H +#define crypto_hash_sha256_H + +#define crypto_hash_sha256_ref_BYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_hash_sha256_ref(const std::string &); +extern "C" { +#endif +extern int crypto_hash_sha256_ref(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hash_sha256 crypto_hash_sha256_ref +#define crypto_hash_sha256_BYTES crypto_hash_sha256_ref_BYTES +#define crypto_hash_sha256_IMPLEMENTATION "crypto_hash/sha256/ref" +#ifndef crypto_hash_sha256_ref_VERSION +#define crypto_hash_sha256_ref_VERSION "-" +#endif +#define crypto_hash_sha256_VERSION crypto_hash_sha256_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h new file mode 100644 index 00000000..fe19d2d9 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h @@ -0,0 +1,22 @@ +#ifndef crypto_hash_sha512_H +#define crypto_hash_sha512_H + +#define crypto_hash_sha512_ref_BYTES 64 +#ifdef __cplusplus +#include +extern std::string crypto_hash_sha512_ref(const std::string &); +extern "C" { +#endif +extern int crypto_hash_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hash_sha512 crypto_hash_sha512_ref +#define crypto_hash_sha512_BYTES crypto_hash_sha512_ref_BYTES +#define crypto_hash_sha512_IMPLEMENTATION "crypto_hash/sha512/ref" +#ifndef crypto_hash_sha512_ref_VERSION +#define crypto_hash_sha512_ref_VERSION "-" +#endif +#define crypto_hash_sha512_VERSION crypto_hash_sha512_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h new file mode 100644 index 00000000..3b473e6c --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h @@ -0,0 +1,23 @@ +#ifndef crypto_hashblocks_sha256_H +#define crypto_hashblocks_sha256_H + +#define crypto_hashblocks_sha256_inplace_STATEBYTES 32 +#define crypto_hashblocks_sha256_inplace_BLOCKBYTES 64 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_hashblocks_sha256_inplace(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hashblocks_sha256 crypto_hashblocks_sha256_inplace +#define crypto_hashblocks_sha256_STATEBYTES crypto_hashblocks_sha256_inplace_STATEBYTES +#define crypto_hashblocks_sha256_BLOCKBYTES crypto_hashblocks_sha256_inplace_BLOCKBYTES +#define crypto_hashblocks_sha256_IMPLEMENTATION "crypto_hashblocks/sha256/inplace" +#ifndef crypto_hashblocks_sha256_inplace_VERSION +#define crypto_hashblocks_sha256_inplace_VERSION "-" +#endif +#define crypto_hashblocks_sha256_VERSION crypto_hashblocks_sha256_inplace_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h new file mode 100644 index 00000000..f66edd09 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h @@ -0,0 +1,23 @@ +#ifndef crypto_hashblocks_sha512_H +#define crypto_hashblocks_sha512_H + +#define crypto_hashblocks_sha512_ref_STATEBYTES 64 +#define crypto_hashblocks_sha512_ref_BLOCKBYTES 128 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_hashblocks_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hashblocks_sha512 crypto_hashblocks_sha512_ref +#define crypto_hashblocks_sha512_STATEBYTES crypto_hashblocks_sha512_ref_STATEBYTES +#define crypto_hashblocks_sha512_BLOCKBYTES crypto_hashblocks_sha512_ref_BLOCKBYTES +#define crypto_hashblocks_sha512_IMPLEMENTATION "crypto_hashblocks/sha512/ref" +#ifndef crypto_hashblocks_sha512_ref_VERSION +#define crypto_hashblocks_sha512_ref_VERSION "-" +#endif +#define crypto_hashblocks_sha512_VERSION crypto_hashblocks_sha512_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h new file mode 100644 index 00000000..de08dc9f --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h @@ -0,0 +1,27 @@ +#ifndef crypto_onetimeauth_poly1305_H +#define crypto_onetimeauth_poly1305_H + +#define crypto_onetimeauth_poly1305_53_BYTES 16 +#define crypto_onetimeauth_poly1305_53_KEYBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_onetimeauth_poly1305_53(const std::string &,const std::string &); +extern void crypto_onetimeauth_poly1305_53_verify(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_onetimeauth_poly1305_53(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_onetimeauth_poly1305_53_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_onetimeauth_poly1305 crypto_onetimeauth_poly1305_53 +#define crypto_onetimeauth_poly1305_verify crypto_onetimeauth_poly1305_53_verify +#define crypto_onetimeauth_poly1305_BYTES crypto_onetimeauth_poly1305_53_BYTES +#define crypto_onetimeauth_poly1305_KEYBYTES crypto_onetimeauth_poly1305_53_KEYBYTES +#define crypto_onetimeauth_poly1305_IMPLEMENTATION "crypto_onetimeauth/poly1305/53" +#ifndef crypto_onetimeauth_poly1305_53_VERSION +#define crypto_onetimeauth_poly1305_53_VERSION "-" +#endif +#define crypto_onetimeauth_poly1305_VERSION crypto_onetimeauth_poly1305_53_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h new file mode 100644 index 00000000..550c4e3d --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h @@ -0,0 +1,27 @@ +#ifndef crypto_scalarmult_curve25519_H +#define crypto_scalarmult_curve25519_H + +#define crypto_scalarmult_curve25519_donna_c64_BYTES 32 +#define crypto_scalarmult_curve25519_donna_c64_SCALARBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_scalarmult_curve25519_donna_c64(const std::string &,const std::string &); +extern std::string crypto_scalarmult_curve25519_donna_c64_base(const std::string &); +extern "C" { +#endif +extern int crypto_scalarmult_curve25519_donna_c64(unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_scalarmult_curve25519_donna_c64_base(unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_scalarmult_curve25519 crypto_scalarmult_curve25519_donna_c64 +#define crypto_scalarmult_curve25519_base crypto_scalarmult_curve25519_donna_c64_base +#define crypto_scalarmult_curve25519_BYTES crypto_scalarmult_curve25519_donna_c64_BYTES +#define crypto_scalarmult_curve25519_SCALARBYTES crypto_scalarmult_curve25519_donna_c64_SCALARBYTES +#define crypto_scalarmult_curve25519_IMPLEMENTATION "crypto_scalarmult/curve25519/donna_c64" +#ifndef crypto_scalarmult_curve25519_donna_c64_VERSION +#define crypto_scalarmult_curve25519_donna_c64_VERSION "-" +#endif +#define crypto_scalarmult_curve25519_VERSION crypto_scalarmult_curve25519_donna_c64_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h new file mode 100644 index 00000000..c930b6f1 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h @@ -0,0 +1,31 @@ +#ifndef crypto_secretbox_xsalsa20poly1305_H +#define crypto_secretbox_xsalsa20poly1305_H + +#define crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES 32 +#define crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES 24 +#define crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES 32 +#define crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES 16 +#ifdef __cplusplus +#include +extern std::string crypto_secretbox_xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &); +extern std::string crypto_secretbox_xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_secretbox_xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_secretbox_xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_secretbox_xsalsa20poly1305 crypto_secretbox_xsalsa20poly1305_ref +#define crypto_secretbox_xsalsa20poly1305_open crypto_secretbox_xsalsa20poly1305_ref_open +#define crypto_secretbox_xsalsa20poly1305_KEYBYTES crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES +#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES +#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES +#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES +#define crypto_secretbox_xsalsa20poly1305_IMPLEMENTATION "crypto_secretbox/xsalsa20poly1305/ref" +#ifndef crypto_secretbox_xsalsa20poly1305_ref_VERSION +#define crypto_secretbox_xsalsa20poly1305_ref_VERSION "-" +#endif +#define crypto_secretbox_xsalsa20poly1305_VERSION crypto_secretbox_xsalsa20poly1305_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h new file mode 100644 index 00000000..936108ef --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h @@ -0,0 +1,32 @@ +#ifndef crypto_sign_edwards25519sha512batch_H +#define crypto_sign_edwards25519sha512batch_H + +#define crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES 64 +#define crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES 32 +#define crypto_sign_edwards25519sha512batch_ref_BYTES 64 +#ifdef __cplusplus +#include +extern std::string crypto_sign_edwards25519sha512batch_ref(const std::string &,const std::string &); +extern std::string crypto_sign_edwards25519sha512batch_ref_open(const std::string &,const std::string &); +extern std::string crypto_sign_edwards25519sha512batch_ref_keypair(std::string *); +extern "C" { +#endif +extern int crypto_sign_edwards25519sha512batch_ref(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_sign_edwards25519sha512batch_ref_open(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_sign_edwards25519sha512batch_ref_keypair(unsigned char *,unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_sign_edwards25519sha512batch crypto_sign_edwards25519sha512batch_ref +#define crypto_sign_edwards25519sha512batch_open crypto_sign_edwards25519sha512batch_ref_open +#define crypto_sign_edwards25519sha512batch_keypair crypto_sign_edwards25519sha512batch_ref_keypair +#define crypto_sign_edwards25519sha512batch_BYTES crypto_sign_edwards25519sha512batch_ref_BYTES +#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES +#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES +#define crypto_sign_edwards25519sha512batch_IMPLEMENTATION "crypto_sign/edwards25519sha512batch/ref" +#ifndef crypto_sign_edwards25519sha512batch_ref_VERSION +#define crypto_sign_edwards25519sha512batch_ref_VERSION "-" +#endif +#define crypto_sign_edwards25519sha512batch_VERSION crypto_sign_edwards25519sha512batch_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h new file mode 100644 index 00000000..76bf9137 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h @@ -0,0 +1,35 @@ +#ifndef crypto_stream_aes128ctr_H +#define crypto_stream_aes128ctr_H + +#define crypto_stream_aes128ctr_portable_KEYBYTES 16 +#define crypto_stream_aes128ctr_portable_NONCEBYTES 16 +#define crypto_stream_aes128ctr_portable_BEFORENMBYTES 1408 +#ifdef __cplusplus +#include +extern std::string crypto_stream_aes128ctr_portable(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_aes128ctr_portable_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_aes128ctr_portable(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_aes128ctr crypto_stream_aes128ctr_portable +#define crypto_stream_aes128ctr_xor crypto_stream_aes128ctr_portable_xor +#define crypto_stream_aes128ctr_beforenm crypto_stream_aes128ctr_portable_beforenm +#define crypto_stream_aes128ctr_afternm crypto_stream_aes128ctr_portable_afternm +#define crypto_stream_aes128ctr_xor_afternm crypto_stream_aes128ctr_portable_xor_afternm +#define crypto_stream_aes128ctr_KEYBYTES crypto_stream_aes128ctr_portable_KEYBYTES +#define crypto_stream_aes128ctr_NONCEBYTES crypto_stream_aes128ctr_portable_NONCEBYTES +#define crypto_stream_aes128ctr_BEFORENMBYTES crypto_stream_aes128ctr_portable_BEFORENMBYTES +#define crypto_stream_aes128ctr_IMPLEMENTATION "crypto_stream/aes128ctr/portable" +#ifndef crypto_stream_aes128ctr_portable_VERSION +#define crypto_stream_aes128ctr_portable_VERSION "-" +#endif +#define crypto_stream_aes128ctr_VERSION crypto_stream_aes128ctr_portable_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h new file mode 100644 index 00000000..c96d20b4 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_salsa20_H +#define crypto_stream_salsa20_H + +#define crypto_stream_salsa20_amd64_xmm6_KEYBYTES 32 +#define crypto_stream_salsa20_amd64_xmm6_NONCEBYTES 8 +#ifdef __cplusplus +#include +extern std::string crypto_stream_salsa20_amd64_xmm6(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_salsa20_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_salsa20_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_salsa20 crypto_stream_salsa20_amd64_xmm6 +#define crypto_stream_salsa20_xor crypto_stream_salsa20_amd64_xmm6_xor +#define crypto_stream_salsa20_beforenm crypto_stream_salsa20_amd64_xmm6_beforenm +#define crypto_stream_salsa20_afternm crypto_stream_salsa20_amd64_xmm6_afternm +#define crypto_stream_salsa20_xor_afternm crypto_stream_salsa20_amd64_xmm6_xor_afternm +#define crypto_stream_salsa20_KEYBYTES crypto_stream_salsa20_amd64_xmm6_KEYBYTES +#define crypto_stream_salsa20_NONCEBYTES crypto_stream_salsa20_amd64_xmm6_NONCEBYTES +#define crypto_stream_salsa20_BEFORENMBYTES crypto_stream_salsa20_amd64_xmm6_BEFORENMBYTES +#define crypto_stream_salsa20_IMPLEMENTATION "crypto_stream/salsa20/amd64_xmm6" +#ifndef crypto_stream_salsa20_amd64_xmm6_VERSION +#define crypto_stream_salsa20_amd64_xmm6_VERSION "-" +#endif +#define crypto_stream_salsa20_VERSION crypto_stream_salsa20_amd64_xmm6_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h new file mode 100644 index 00000000..051e4e39 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_salsa2012_H +#define crypto_stream_salsa2012_H + +#define crypto_stream_salsa2012_amd64_xmm6_KEYBYTES 32 +#define crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES 8 +#ifdef __cplusplus +#include +extern std::string crypto_stream_salsa2012_amd64_xmm6(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_salsa2012_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_salsa2012 crypto_stream_salsa2012_amd64_xmm6 +#define crypto_stream_salsa2012_xor crypto_stream_salsa2012_amd64_xmm6_xor +#define crypto_stream_salsa2012_beforenm crypto_stream_salsa2012_amd64_xmm6_beforenm +#define crypto_stream_salsa2012_afternm crypto_stream_salsa2012_amd64_xmm6_afternm +#define crypto_stream_salsa2012_xor_afternm crypto_stream_salsa2012_amd64_xmm6_xor_afternm +#define crypto_stream_salsa2012_KEYBYTES crypto_stream_salsa2012_amd64_xmm6_KEYBYTES +#define crypto_stream_salsa2012_NONCEBYTES crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES +#define crypto_stream_salsa2012_BEFORENMBYTES crypto_stream_salsa2012_amd64_xmm6_BEFORENMBYTES +#define crypto_stream_salsa2012_IMPLEMENTATION "crypto_stream/salsa2012/amd64_xmm6" +#ifndef crypto_stream_salsa2012_amd64_xmm6_VERSION +#define crypto_stream_salsa2012_amd64_xmm6_VERSION "-" +#endif +#define crypto_stream_salsa2012_VERSION crypto_stream_salsa2012_amd64_xmm6_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h new file mode 100644 index 00000000..4bd470c3 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_salsa208_H +#define crypto_stream_salsa208_H + +#define crypto_stream_salsa208_amd64_xmm6_KEYBYTES 32 +#define crypto_stream_salsa208_amd64_xmm6_NONCEBYTES 8 +#ifdef __cplusplus +#include +extern std::string crypto_stream_salsa208_amd64_xmm6(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_salsa208_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_salsa208_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_salsa208 crypto_stream_salsa208_amd64_xmm6 +#define crypto_stream_salsa208_xor crypto_stream_salsa208_amd64_xmm6_xor +#define crypto_stream_salsa208_beforenm crypto_stream_salsa208_amd64_xmm6_beforenm +#define crypto_stream_salsa208_afternm crypto_stream_salsa208_amd64_xmm6_afternm +#define crypto_stream_salsa208_xor_afternm crypto_stream_salsa208_amd64_xmm6_xor_afternm +#define crypto_stream_salsa208_KEYBYTES crypto_stream_salsa208_amd64_xmm6_KEYBYTES +#define crypto_stream_salsa208_NONCEBYTES crypto_stream_salsa208_amd64_xmm6_NONCEBYTES +#define crypto_stream_salsa208_BEFORENMBYTES crypto_stream_salsa208_amd64_xmm6_BEFORENMBYTES +#define crypto_stream_salsa208_IMPLEMENTATION "crypto_stream/salsa208/amd64_xmm6" +#ifndef crypto_stream_salsa208_amd64_xmm6_VERSION +#define crypto_stream_salsa208_amd64_xmm6_VERSION "-" +#endif +#define crypto_stream_salsa208_VERSION crypto_stream_salsa208_amd64_xmm6_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h new file mode 100644 index 00000000..d75268c6 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_xsalsa20_H +#define crypto_stream_xsalsa20_H + +#define crypto_stream_xsalsa20_ref_KEYBYTES 32 +#define crypto_stream_xsalsa20_ref_NONCEBYTES 24 +#ifdef __cplusplus +#include +extern std::string crypto_stream_xsalsa20_ref(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_xsalsa20_ref_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_xsalsa20_ref(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_xsalsa20 crypto_stream_xsalsa20_ref +#define crypto_stream_xsalsa20_xor crypto_stream_xsalsa20_ref_xor +#define crypto_stream_xsalsa20_beforenm crypto_stream_xsalsa20_ref_beforenm +#define crypto_stream_xsalsa20_afternm crypto_stream_xsalsa20_ref_afternm +#define crypto_stream_xsalsa20_xor_afternm crypto_stream_xsalsa20_ref_xor_afternm +#define crypto_stream_xsalsa20_KEYBYTES crypto_stream_xsalsa20_ref_KEYBYTES +#define crypto_stream_xsalsa20_NONCEBYTES crypto_stream_xsalsa20_ref_NONCEBYTES +#define crypto_stream_xsalsa20_BEFORENMBYTES crypto_stream_xsalsa20_ref_BEFORENMBYTES +#define crypto_stream_xsalsa20_IMPLEMENTATION "crypto_stream/xsalsa20/ref" +#ifndef crypto_stream_xsalsa20_ref_VERSION +#define crypto_stream_xsalsa20_ref_VERSION "-" +#endif +#define crypto_stream_xsalsa20_VERSION crypto_stream_xsalsa20_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h new file mode 100644 index 00000000..b0ce9656 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h @@ -0,0 +1,11 @@ +#ifndef crypto_types_h +#define crypto_types_h +typedef short crypto_int16; +typedef int crypto_int32; +typedef long long crypto_int64; +typedef signed char crypto_int8; +typedef unsigned short crypto_uint16; +typedef unsigned int crypto_uint32; +typedef unsigned long long crypto_uint64; +typedef unsigned char crypto_uint8; +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h new file mode 100644 index 00000000..6bf6ca11 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h @@ -0,0 +1,21 @@ +#ifndef crypto_verify_16_H +#define crypto_verify_16_H + +#define crypto_verify_16_ref_BYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_verify_16_ref(const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_verify_16 crypto_verify_16_ref +#define crypto_verify_16_BYTES crypto_verify_16_ref_BYTES +#define crypto_verify_16_IMPLEMENTATION "crypto_verify/16/ref" +#ifndef crypto_verify_16_ref_VERSION +#define crypto_verify_16_ref_VERSION "-" +#endif +#define crypto_verify_16_VERSION crypto_verify_16_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h new file mode 100644 index 00000000..bd5fc644 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h @@ -0,0 +1,21 @@ +#ifndef crypto_verify_32_H +#define crypto_verify_32_H + +#define crypto_verify_32_ref_BYTES 32 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_verify_32_ref(const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_verify_32 crypto_verify_32_ref +#define crypto_verify_32_BYTES crypto_verify_32_ref_BYTES +#define crypto_verify_32_IMPLEMENTATION "crypto_verify/32/ref" +#ifndef crypto_verify_32_ref_VERSION +#define crypto_verify_32_ref_VERSION "-" +#endif +#define crypto_verify_32_VERSION crypto_verify_32_ref_VERSION + +#endif diff --git a/make-mac.mk b/make-mac.mk index 8ff1b772..b71ca2fe 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,6 +33,12 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif +# Use precompiled extremely fast Salsa20/12 from "cnacl" included in ext/bin +# See https://github.com/cjdelisle/cnacl +DEFS+=-DZT_USE_LIBSODIUM +CFLAGS+=-Iext/bin/cnacl-osx-amd64/include +LIBS+=ext/bin/cnacl-osx-amd64/libnacl.a + ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif diff --git a/node/Identity.cpp b/node/Identity.cpp index 89fdb836..d1b21e9c 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -45,7 +45,7 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub // ordinary Salsa20 is randomly seekable. This is good for a cipher // but is not what we want for sequential memory-harndess. memset(genmem,0,ZT_IDENTITY_GEN_MEMORY); - Salsa20 s20(digest,256,(char *)digest + 32); + Salsa20 s20(digest,(char *)digest + 32); s20.crypt20((char *)genmem,(char *)genmem,64); for(unsigned long i=64;i> (32 - (c)))) #define XOR(v,w) ((v) ^ (w)) #define PLUS(v,w) ((uint32_t)((v) + (w))) @@ -66,8 +68,7 @@ static const _s20sseconsts _S20SSECONSTANTS; namespace ZeroTier { -void Salsa20::init(const void *key,unsigned int kbits,const void *iv) - throw() +void Salsa20::init(const void *key,const void *iv) { #ifdef ZT_SALSA20_SSE const uint32_t *k = (const uint32_t *)key; @@ -78,14 +79,9 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) _state.i[10] = k[1]; _state.i[7] = k[2]; _state.i[4] = k[3]; - if (kbits == 256) { - k += 4; - _state.i[1] = 0x3320646e; - _state.i[2] = 0x79622d32; - } else { - _state.i[1] = 0x3120646e; - _state.i[2] = 0x79622d36; - } + k += 4; + _state.i[1] = 0x3320646e; + _state.i[2] = 0x79622d32; _state.i[15] = k[0]; _state.i[12] = k[1]; _state.i[9] = k[2]; @@ -95,19 +91,14 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) _state.i[5] = 0; _state.i[8] = 0; #else - const char *constants; + const char *const constants = "expand 32-byte k"; const uint8_t *k = (const uint8_t *)key; _state.i[1] = U8TO32_LITTLE(k + 0); _state.i[2] = U8TO32_LITTLE(k + 4); _state.i[3] = U8TO32_LITTLE(k + 8); _state.i[4] = U8TO32_LITTLE(k + 12); - if (kbits == 256) { /* recommended */ - k += 16; - constants = "expand 32-byte k"; - } else { /* kbits == 128 */ - constants = "expand 16-byte k"; - } + k += 16; _state.i[5] = U8TO32_LITTLE(constants + 4); _state.i[6] = U8TO32_LITTLE(((const uint8_t *)iv) + 0); _state.i[7] = U8TO32_LITTLE(((const uint8_t *)iv) + 4); @@ -124,7 +115,6 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) } void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) - throw() { uint8_t tmp[64]; const uint8_t *m = (const uint8_t *)in; @@ -624,7 +614,6 @@ void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) } void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) - throw() { uint8_t tmp[64]; const uint8_t *m = (const uint8_t *)in; @@ -1356,3 +1345,5 @@ void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) } } // namespace ZeroTier + +#endif // !ZT_USE_LIBSODIUM diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 6405d450..5e4c68be 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -10,10 +10,82 @@ #include #include #include +#include #include "Constants.hpp" #include "Utils.hpp" +#ifdef ZT_USE_LIBSODIUM + +#include +#include + +namespace ZeroTier { + +/** + * Salsa20 stream cipher + */ +class Salsa20 +{ +public: + Salsa20() {} + ~Salsa20() { Utils::burn(_k,sizeof(_k)); } + + /** + * @param key 256-bit (32 byte) key + * @param iv 64-bit initialization vector + */ + Salsa20(const void *key,const void *iv) + { + memcpy(_k,key,32); + memcpy(&_iv,iv,8); + } + + /** + * Initialize cipher + * + * @param key Key bits + * @param iv 64-bit initialization vector + */ + inline void init(const void *key,const void *iv) + { + memcpy(_k,key,32); + memcpy(&_iv,iv,8); + } + + /** + * Encrypt/decrypt data using Salsa20/12 + * + * @param in Input data + * @param out Output buffer + * @param bytes Length of data + */ + inline void crypt12(const void *in,void *out,unsigned int bytes) + { + crypto_stream_salsa2012_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); + } + + /** + * Encrypt/decrypt data using Salsa20/20 + * + * @param in Input data + * @param out Output buffer + * @param bytes Length of data + */ + inline void crypt20(const void *in,void *out,unsigned int bytes) + { + crypto_stream_salsa20_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); + } + +private: + uint64_t _k[4]; + uint64_t _iv; +}; + +} // namespace ZeroTier + +#else // !ZT_USE_LIBSODIUM + #if (!defined(ZT_SALSA20_SSE)) && (defined(__SSE2__) || defined(__WINDOWS__)) #define ZT_SALSA20_SSE 1 #endif @@ -30,30 +102,25 @@ namespace ZeroTier { class Salsa20 { public: - Salsa20() throw() {} - + Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } /** - * @param key Key bits - * @param kbits Number of key bits: 128 or 256 (recommended) + * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector */ - Salsa20(const void *key,unsigned int kbits,const void *iv) - throw() + Salsa20(const void *key,const void *iv) { - init(key,kbits,iv); + init(key,iv); } /** * Initialize cipher * * @param key Key bits - * @param kbits Number of key bits: 128 or 256 (recommended) * @param iv 64-bit initialization vector */ - void init(const void *key,unsigned int kbits,const void *iv) - throw(); + void init(const void *key,const void *iv); /** * Encrypt/decrypt data using Salsa20/12 @@ -62,8 +129,7 @@ public: * @param out Output buffer * @param bytes Length of data */ - void crypt12(const void *in,void *out,unsigned int bytes) - throw(); + void crypt12(const void *in,void *out,unsigned int bytes); /** * Encrypt/decrypt data using Salsa20/20 @@ -72,8 +138,7 @@ public: * @param out Output buffer * @param bytes Length of data */ - void crypt20(const void *in,void *out,unsigned int bytes) - throw(); + void crypt20(const void *in,void *out,unsigned int bytes); private: union { @@ -86,4 +151,6 @@ private: } // namespace ZeroTier +#endif // ZT_USE_LIBSODIUM + #endif diff --git a/node/Utils.cpp b/node/Utils.cpp index fb448dd6..92d14d19 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -156,7 +156,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) s20Key[1] = (uint64_t)buf; // address of buf s20Key[2] = (uint64_t)s20Key; // address of s20Key[] s20Key[3] = (uint64_t)&s20; // address of s20 - s20.init(s20Key,256,s20Key); + s20.init(s20Key,s20Key); } #ifdef __WINDOWS__ diff --git a/selftest.cpp b/selftest.cpp index 48625d53..fe0aa933 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -153,16 +153,16 @@ static int testCrypto() memset(buf2,0,sizeof(buf2)); memset(buf3,0,sizeof(buf3)); Salsa20 s20; - s20.init("12345678123456781234567812345678",256,"12345678"); + s20.init("12345678123456781234567812345678","12345678"); s20.crypt20(buf1,buf2,sizeof(buf1)); - s20.init("12345678123456781234567812345678",256,"12345678"); + s20.init("12345678123456781234567812345678","12345678"); s20.crypt20(buf2,buf3,sizeof(buf2)); if (memcmp(buf1,buf3,sizeof(buf1))) { std::cout << "FAIL (encrypt/decrypt test)" << std::endl; return -1; } } - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); s20.crypt20(buf1,buf2,64); @@ -170,7 +170,7 @@ static int testCrypto() std::cout << "FAIL (test vector 0)" << std::endl; return -1; } - s20.init(s2012TV0Key,256,s2012TV0Iv); + s20.init(s2012TV0Key,s2012TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); s20.crypt12(buf1,buf2,64); @@ -191,7 +191,7 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { @@ -209,7 +209,7 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { -- cgit v1.2.3 From 7a94f6305812b7ea5748283a6ec9503f4ea9c7e1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 17:54:12 -0700 Subject: Back out NaCl since the old one with xmm6 salsa2012 does not support multi-block use and the new one is slower. --- ext/bin/cnacl-osx-amd64/README.md | 53 -------------- .../include/sodium/crypto_auth_hmacsha256.h | 27 ------- .../include/sodium/crypto_auth_hmacsha512256.h | 27 ------- .../sodium/crypto_box_curve25519xsalsa20poly1305.h | 44 ------------ .../include/sodium/crypto_core_hsalsa20.h | 27 ------- .../include/sodium/crypto_core_salsa20.h | 27 ------- .../include/sodium/crypto_core_salsa2012.h | 27 ------- .../include/sodium/crypto_core_salsa208.h | 27 ------- .../include/sodium/crypto_hash_sha256.h | 22 ------ .../include/sodium/crypto_hash_sha512.h | 22 ------ .../include/sodium/crypto_hashblocks_sha256.h | 23 ------ .../include/sodium/crypto_hashblocks_sha512.h | 23 ------ .../include/sodium/crypto_onetimeauth_poly1305.h | 27 ------- .../include/sodium/crypto_scalarmult_curve25519.h | 27 ------- .../sodium/crypto_secretbox_xsalsa20poly1305.h | 31 -------- .../sodium/crypto_sign_edwards25519sha512batch.h | 32 --------- .../include/sodium/crypto_stream_aes128ctr.h | 35 --------- .../include/sodium/crypto_stream_salsa20.h | 34 --------- .../include/sodium/crypto_stream_salsa2012.h | 34 --------- .../include/sodium/crypto_stream_salsa208.h | 34 --------- .../include/sodium/crypto_stream_xsalsa20.h | 34 --------- .../cnacl-osx-amd64/include/sodium/crypto_types.h | 11 --- .../include/sodium/crypto_verify_16.h | 21 ------ .../include/sodium/crypto_verify_32.h | 21 ------ ext/bin/cnacl-osx-amd64/libnacl.a | Bin 401672 -> 0 bytes make-mac.mk | 6 -- node/Node.cpp | 22 +++--- node/Node.hpp | 8 +-- node/Salsa20.cpp | 4 -- node/Salsa20.hpp | 78 ++------------------- node/Utils.cpp | 2 + 31 files changed, 18 insertions(+), 792 deletions(-) delete mode 100644 ext/bin/cnacl-osx-amd64/README.md delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h delete mode 100644 ext/bin/cnacl-osx-amd64/libnacl.a (limited to 'node') diff --git a/ext/bin/cnacl-osx-amd64/README.md b/ext/bin/cnacl-osx-amd64/README.md deleted file mode 100644 index 35426286..00000000 --- a/ext/bin/cnacl-osx-amd64/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# cNaCl - - If you would like to be confusing, you could pronounce it sea-salt - -This is a fork NaCl by Daniel J. Bernstein and Tanja Lange. -The build has been ported to cmake so it can be cross compiled and build output is reliable. -Since it uses cmake, it could theoretically be built on windows but this has not been tested. -It does compile using mingw32. - -## How do I make this thing work? - - mkdir cbuild - cd cbuild - cmake .. - make - -## Ok now how about cross compiling? - - mkdir cbuildw32 - cd cbuildw32 - cmake -DCMAKE_TOOLCHAIN_FILE=../CMakeWindows.txt .. - make - -## Why fork? - -NaCl builds using a shell script called `./do`. This script does compiling, testing, measuring -and selection of the best implementation of each algorithm for the given machine. It also generates -the header files which will be used. - -The problems with `./do` are it's slow, it tries compiling with multiple different compiler -profiles, it's very platform independent but it doesn't run on Windows and most importantly, with -compiling, testing and measuring so tightly bound, it is impossible to cross compile for a -different operating system. - - -## How it works - -The first time you build for a new ABI, it will trigger the traditional nacl `./do` script. -What cNaCl does is parse the resulting headers from the `./do` build and create a plan so that it -can repeat roughly the same build. - -If there is already a plan for the given ABI, the build uses this plan and the build is very fast. - -Plans are stored in `./cmake/plans/` and I will be adding plans as I find new ones. - - -## What else is new? - -There is a problem with the `./do` build which prevents it from running on some ARM based machines, -this was fixed by adding a more lax method for measuring CPU speed as a fall back. - - -`#EOF#` diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h deleted file mode 100644 index 6b5600f3..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_auth_hmacsha256_H -#define crypto_auth_hmacsha256_H - -#define crypto_auth_hmacsha256_ref_BYTES 32 -#define crypto_auth_hmacsha256_ref_KEYBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_auth_hmacsha256_ref(const std::string &,const std::string &); -extern void crypto_auth_hmacsha256_ref_verify(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_auth_hmacsha256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_auth_hmacsha256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_auth_hmacsha256 crypto_auth_hmacsha256_ref -#define crypto_auth_hmacsha256_verify crypto_auth_hmacsha256_ref_verify -#define crypto_auth_hmacsha256_BYTES crypto_auth_hmacsha256_ref_BYTES -#define crypto_auth_hmacsha256_KEYBYTES crypto_auth_hmacsha256_ref_KEYBYTES -#define crypto_auth_hmacsha256_IMPLEMENTATION "crypto_auth/hmacsha256/ref" -#ifndef crypto_auth_hmacsha256_ref_VERSION -#define crypto_auth_hmacsha256_ref_VERSION "-" -#endif -#define crypto_auth_hmacsha256_VERSION crypto_auth_hmacsha256_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h deleted file mode 100644 index c9bd96e4..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_auth_hmacsha512256_H -#define crypto_auth_hmacsha512256_H - -#define crypto_auth_hmacsha512256_ref_BYTES 32 -#define crypto_auth_hmacsha512256_ref_KEYBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_auth_hmacsha512256_ref(const std::string &,const std::string &); -extern void crypto_auth_hmacsha512256_ref_verify(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_auth_hmacsha512256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_auth_hmacsha512256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_auth_hmacsha512256 crypto_auth_hmacsha512256_ref -#define crypto_auth_hmacsha512256_verify crypto_auth_hmacsha512256_ref_verify -#define crypto_auth_hmacsha512256_BYTES crypto_auth_hmacsha512256_ref_BYTES -#define crypto_auth_hmacsha512256_KEYBYTES crypto_auth_hmacsha512256_ref_KEYBYTES -#define crypto_auth_hmacsha512256_IMPLEMENTATION "crypto_auth/hmacsha512256/ref" -#ifndef crypto_auth_hmacsha512256_ref_VERSION -#define crypto_auth_hmacsha512256_ref_VERSION "-" -#endif -#define crypto_auth_hmacsha512256_VERSION crypto_auth_hmacsha512256_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h deleted file mode 100644 index e2c3b4cc..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef crypto_box_curve25519xsalsa20poly1305_H -#define crypto_box_curve25519xsalsa20poly1305_H - -#define crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES 24 -#define crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES 16 -#ifdef __cplusplus -#include -extern std::string crypto_box_curve25519xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &,const std::string &); -extern std::string crypto_box_curve25519xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &,const std::string &); -extern std::string crypto_box_curve25519xsalsa20poly1305_ref_keypair(std::string *); -extern "C" { -#endif -extern int crypto_box_curve25519xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_keypair(unsigned char *,unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_beforenm(unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_open_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_box_curve25519xsalsa20poly1305 crypto_box_curve25519xsalsa20poly1305_ref -#define crypto_box_curve25519xsalsa20poly1305_open crypto_box_curve25519xsalsa20poly1305_ref_open -#define crypto_box_curve25519xsalsa20poly1305_keypair crypto_box_curve25519xsalsa20poly1305_ref_keypair -#define crypto_box_curve25519xsalsa20poly1305_beforenm crypto_box_curve25519xsalsa20poly1305_ref_beforenm -#define crypto_box_curve25519xsalsa20poly1305_afternm crypto_box_curve25519xsalsa20poly1305_ref_afternm -#define crypto_box_curve25519xsalsa20poly1305_open_afternm crypto_box_curve25519xsalsa20poly1305_ref_open_afternm -#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES -#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES -#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES -#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES -#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES -#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES -#define crypto_box_curve25519xsalsa20poly1305_IMPLEMENTATION "crypto_box/curve25519xsalsa20poly1305/ref" -#ifndef crypto_box_curve25519xsalsa20poly1305_ref_VERSION -#define crypto_box_curve25519xsalsa20poly1305_ref_VERSION "-" -#endif -#define crypto_box_curve25519xsalsa20poly1305_VERSION crypto_box_curve25519xsalsa20poly1305_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h deleted file mode 100644 index abae188e..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_hsalsa20_H -#define crypto_core_hsalsa20_H - -#define crypto_core_hsalsa20_ref_OUTPUTBYTES 32 -#define crypto_core_hsalsa20_ref_INPUTBYTES 16 -#define crypto_core_hsalsa20_ref_KEYBYTES 32 -#define crypto_core_hsalsa20_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_hsalsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_hsalsa20 crypto_core_hsalsa20_ref -#define crypto_core_hsalsa20_OUTPUTBYTES crypto_core_hsalsa20_ref_OUTPUTBYTES -#define crypto_core_hsalsa20_INPUTBYTES crypto_core_hsalsa20_ref_INPUTBYTES -#define crypto_core_hsalsa20_KEYBYTES crypto_core_hsalsa20_ref_KEYBYTES -#define crypto_core_hsalsa20_CONSTBYTES crypto_core_hsalsa20_ref_CONSTBYTES -#define crypto_core_hsalsa20_IMPLEMENTATION "crypto_core/hsalsa20/ref" -#ifndef crypto_core_hsalsa20_ref_VERSION -#define crypto_core_hsalsa20_ref_VERSION "-" -#endif -#define crypto_core_hsalsa20_VERSION crypto_core_hsalsa20_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h deleted file mode 100644 index 9737b101..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_salsa20_H -#define crypto_core_salsa20_H - -#define crypto_core_salsa20_ref_OUTPUTBYTES 64 -#define crypto_core_salsa20_ref_INPUTBYTES 16 -#define crypto_core_salsa20_ref_KEYBYTES 32 -#define crypto_core_salsa20_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_salsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_salsa20 crypto_core_salsa20_ref -#define crypto_core_salsa20_OUTPUTBYTES crypto_core_salsa20_ref_OUTPUTBYTES -#define crypto_core_salsa20_INPUTBYTES crypto_core_salsa20_ref_INPUTBYTES -#define crypto_core_salsa20_KEYBYTES crypto_core_salsa20_ref_KEYBYTES -#define crypto_core_salsa20_CONSTBYTES crypto_core_salsa20_ref_CONSTBYTES -#define crypto_core_salsa20_IMPLEMENTATION "crypto_core/salsa20/ref" -#ifndef crypto_core_salsa20_ref_VERSION -#define crypto_core_salsa20_ref_VERSION "-" -#endif -#define crypto_core_salsa20_VERSION crypto_core_salsa20_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h deleted file mode 100644 index 137cd47f..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_salsa2012_H -#define crypto_core_salsa2012_H - -#define crypto_core_salsa2012_ref_OUTPUTBYTES 64 -#define crypto_core_salsa2012_ref_INPUTBYTES 16 -#define crypto_core_salsa2012_ref_KEYBYTES 32 -#define crypto_core_salsa2012_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_salsa2012_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_salsa2012 crypto_core_salsa2012_ref -#define crypto_core_salsa2012_OUTPUTBYTES crypto_core_salsa2012_ref_OUTPUTBYTES -#define crypto_core_salsa2012_INPUTBYTES crypto_core_salsa2012_ref_INPUTBYTES -#define crypto_core_salsa2012_KEYBYTES crypto_core_salsa2012_ref_KEYBYTES -#define crypto_core_salsa2012_CONSTBYTES crypto_core_salsa2012_ref_CONSTBYTES -#define crypto_core_salsa2012_IMPLEMENTATION "crypto_core/salsa2012/ref" -#ifndef crypto_core_salsa2012_ref_VERSION -#define crypto_core_salsa2012_ref_VERSION "-" -#endif -#define crypto_core_salsa2012_VERSION crypto_core_salsa2012_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h deleted file mode 100644 index 4895bbbe..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_salsa208_H -#define crypto_core_salsa208_H - -#define crypto_core_salsa208_ref_OUTPUTBYTES 64 -#define crypto_core_salsa208_ref_INPUTBYTES 16 -#define crypto_core_salsa208_ref_KEYBYTES 32 -#define crypto_core_salsa208_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_salsa208_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_salsa208 crypto_core_salsa208_ref -#define crypto_core_salsa208_OUTPUTBYTES crypto_core_salsa208_ref_OUTPUTBYTES -#define crypto_core_salsa208_INPUTBYTES crypto_core_salsa208_ref_INPUTBYTES -#define crypto_core_salsa208_KEYBYTES crypto_core_salsa208_ref_KEYBYTES -#define crypto_core_salsa208_CONSTBYTES crypto_core_salsa208_ref_CONSTBYTES -#define crypto_core_salsa208_IMPLEMENTATION "crypto_core/salsa208/ref" -#ifndef crypto_core_salsa208_ref_VERSION -#define crypto_core_salsa208_ref_VERSION "-" -#endif -#define crypto_core_salsa208_VERSION crypto_core_salsa208_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h deleted file mode 100644 index 20d18703..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef crypto_hash_sha256_H -#define crypto_hash_sha256_H - -#define crypto_hash_sha256_ref_BYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_hash_sha256_ref(const std::string &); -extern "C" { -#endif -extern int crypto_hash_sha256_ref(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hash_sha256 crypto_hash_sha256_ref -#define crypto_hash_sha256_BYTES crypto_hash_sha256_ref_BYTES -#define crypto_hash_sha256_IMPLEMENTATION "crypto_hash/sha256/ref" -#ifndef crypto_hash_sha256_ref_VERSION -#define crypto_hash_sha256_ref_VERSION "-" -#endif -#define crypto_hash_sha256_VERSION crypto_hash_sha256_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h deleted file mode 100644 index fe19d2d9..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef crypto_hash_sha512_H -#define crypto_hash_sha512_H - -#define crypto_hash_sha512_ref_BYTES 64 -#ifdef __cplusplus -#include -extern std::string crypto_hash_sha512_ref(const std::string &); -extern "C" { -#endif -extern int crypto_hash_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hash_sha512 crypto_hash_sha512_ref -#define crypto_hash_sha512_BYTES crypto_hash_sha512_ref_BYTES -#define crypto_hash_sha512_IMPLEMENTATION "crypto_hash/sha512/ref" -#ifndef crypto_hash_sha512_ref_VERSION -#define crypto_hash_sha512_ref_VERSION "-" -#endif -#define crypto_hash_sha512_VERSION crypto_hash_sha512_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h deleted file mode 100644 index 3b473e6c..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef crypto_hashblocks_sha256_H -#define crypto_hashblocks_sha256_H - -#define crypto_hashblocks_sha256_inplace_STATEBYTES 32 -#define crypto_hashblocks_sha256_inplace_BLOCKBYTES 64 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_hashblocks_sha256_inplace(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hashblocks_sha256 crypto_hashblocks_sha256_inplace -#define crypto_hashblocks_sha256_STATEBYTES crypto_hashblocks_sha256_inplace_STATEBYTES -#define crypto_hashblocks_sha256_BLOCKBYTES crypto_hashblocks_sha256_inplace_BLOCKBYTES -#define crypto_hashblocks_sha256_IMPLEMENTATION "crypto_hashblocks/sha256/inplace" -#ifndef crypto_hashblocks_sha256_inplace_VERSION -#define crypto_hashblocks_sha256_inplace_VERSION "-" -#endif -#define crypto_hashblocks_sha256_VERSION crypto_hashblocks_sha256_inplace_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h deleted file mode 100644 index f66edd09..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef crypto_hashblocks_sha512_H -#define crypto_hashblocks_sha512_H - -#define crypto_hashblocks_sha512_ref_STATEBYTES 64 -#define crypto_hashblocks_sha512_ref_BLOCKBYTES 128 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_hashblocks_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hashblocks_sha512 crypto_hashblocks_sha512_ref -#define crypto_hashblocks_sha512_STATEBYTES crypto_hashblocks_sha512_ref_STATEBYTES -#define crypto_hashblocks_sha512_BLOCKBYTES crypto_hashblocks_sha512_ref_BLOCKBYTES -#define crypto_hashblocks_sha512_IMPLEMENTATION "crypto_hashblocks/sha512/ref" -#ifndef crypto_hashblocks_sha512_ref_VERSION -#define crypto_hashblocks_sha512_ref_VERSION "-" -#endif -#define crypto_hashblocks_sha512_VERSION crypto_hashblocks_sha512_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h deleted file mode 100644 index de08dc9f..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_onetimeauth_poly1305_H -#define crypto_onetimeauth_poly1305_H - -#define crypto_onetimeauth_poly1305_53_BYTES 16 -#define crypto_onetimeauth_poly1305_53_KEYBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_onetimeauth_poly1305_53(const std::string &,const std::string &); -extern void crypto_onetimeauth_poly1305_53_verify(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_onetimeauth_poly1305_53(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_onetimeauth_poly1305_53_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_onetimeauth_poly1305 crypto_onetimeauth_poly1305_53 -#define crypto_onetimeauth_poly1305_verify crypto_onetimeauth_poly1305_53_verify -#define crypto_onetimeauth_poly1305_BYTES crypto_onetimeauth_poly1305_53_BYTES -#define crypto_onetimeauth_poly1305_KEYBYTES crypto_onetimeauth_poly1305_53_KEYBYTES -#define crypto_onetimeauth_poly1305_IMPLEMENTATION "crypto_onetimeauth/poly1305/53" -#ifndef crypto_onetimeauth_poly1305_53_VERSION -#define crypto_onetimeauth_poly1305_53_VERSION "-" -#endif -#define crypto_onetimeauth_poly1305_VERSION crypto_onetimeauth_poly1305_53_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h deleted file mode 100644 index 550c4e3d..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_scalarmult_curve25519_H -#define crypto_scalarmult_curve25519_H - -#define crypto_scalarmult_curve25519_donna_c64_BYTES 32 -#define crypto_scalarmult_curve25519_donna_c64_SCALARBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_scalarmult_curve25519_donna_c64(const std::string &,const std::string &); -extern std::string crypto_scalarmult_curve25519_donna_c64_base(const std::string &); -extern "C" { -#endif -extern int crypto_scalarmult_curve25519_donna_c64(unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_scalarmult_curve25519_donna_c64_base(unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_scalarmult_curve25519 crypto_scalarmult_curve25519_donna_c64 -#define crypto_scalarmult_curve25519_base crypto_scalarmult_curve25519_donna_c64_base -#define crypto_scalarmult_curve25519_BYTES crypto_scalarmult_curve25519_donna_c64_BYTES -#define crypto_scalarmult_curve25519_SCALARBYTES crypto_scalarmult_curve25519_donna_c64_SCALARBYTES -#define crypto_scalarmult_curve25519_IMPLEMENTATION "crypto_scalarmult/curve25519/donna_c64" -#ifndef crypto_scalarmult_curve25519_donna_c64_VERSION -#define crypto_scalarmult_curve25519_donna_c64_VERSION "-" -#endif -#define crypto_scalarmult_curve25519_VERSION crypto_scalarmult_curve25519_donna_c64_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h deleted file mode 100644 index c930b6f1..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef crypto_secretbox_xsalsa20poly1305_H -#define crypto_secretbox_xsalsa20poly1305_H - -#define crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES 32 -#define crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES 24 -#define crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES 32 -#define crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES 16 -#ifdef __cplusplus -#include -extern std::string crypto_secretbox_xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &); -extern std::string crypto_secretbox_xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_secretbox_xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_secretbox_xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_secretbox_xsalsa20poly1305 crypto_secretbox_xsalsa20poly1305_ref -#define crypto_secretbox_xsalsa20poly1305_open crypto_secretbox_xsalsa20poly1305_ref_open -#define crypto_secretbox_xsalsa20poly1305_KEYBYTES crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES -#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES -#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES -#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES -#define crypto_secretbox_xsalsa20poly1305_IMPLEMENTATION "crypto_secretbox/xsalsa20poly1305/ref" -#ifndef crypto_secretbox_xsalsa20poly1305_ref_VERSION -#define crypto_secretbox_xsalsa20poly1305_ref_VERSION "-" -#endif -#define crypto_secretbox_xsalsa20poly1305_VERSION crypto_secretbox_xsalsa20poly1305_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h deleted file mode 100644 index 936108ef..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef crypto_sign_edwards25519sha512batch_H -#define crypto_sign_edwards25519sha512batch_H - -#define crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES 64 -#define crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES 32 -#define crypto_sign_edwards25519sha512batch_ref_BYTES 64 -#ifdef __cplusplus -#include -extern std::string crypto_sign_edwards25519sha512batch_ref(const std::string &,const std::string &); -extern std::string crypto_sign_edwards25519sha512batch_ref_open(const std::string &,const std::string &); -extern std::string crypto_sign_edwards25519sha512batch_ref_keypair(std::string *); -extern "C" { -#endif -extern int crypto_sign_edwards25519sha512batch_ref(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_sign_edwards25519sha512batch_ref_open(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_sign_edwards25519sha512batch_ref_keypair(unsigned char *,unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_sign_edwards25519sha512batch crypto_sign_edwards25519sha512batch_ref -#define crypto_sign_edwards25519sha512batch_open crypto_sign_edwards25519sha512batch_ref_open -#define crypto_sign_edwards25519sha512batch_keypair crypto_sign_edwards25519sha512batch_ref_keypair -#define crypto_sign_edwards25519sha512batch_BYTES crypto_sign_edwards25519sha512batch_ref_BYTES -#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES -#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES -#define crypto_sign_edwards25519sha512batch_IMPLEMENTATION "crypto_sign/edwards25519sha512batch/ref" -#ifndef crypto_sign_edwards25519sha512batch_ref_VERSION -#define crypto_sign_edwards25519sha512batch_ref_VERSION "-" -#endif -#define crypto_sign_edwards25519sha512batch_VERSION crypto_sign_edwards25519sha512batch_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h deleted file mode 100644 index 76bf9137..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef crypto_stream_aes128ctr_H -#define crypto_stream_aes128ctr_H - -#define crypto_stream_aes128ctr_portable_KEYBYTES 16 -#define crypto_stream_aes128ctr_portable_NONCEBYTES 16 -#define crypto_stream_aes128ctr_portable_BEFORENMBYTES 1408 -#ifdef __cplusplus -#include -extern std::string crypto_stream_aes128ctr_portable(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_aes128ctr_portable_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_aes128ctr_portable(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_aes128ctr crypto_stream_aes128ctr_portable -#define crypto_stream_aes128ctr_xor crypto_stream_aes128ctr_portable_xor -#define crypto_stream_aes128ctr_beforenm crypto_stream_aes128ctr_portable_beforenm -#define crypto_stream_aes128ctr_afternm crypto_stream_aes128ctr_portable_afternm -#define crypto_stream_aes128ctr_xor_afternm crypto_stream_aes128ctr_portable_xor_afternm -#define crypto_stream_aes128ctr_KEYBYTES crypto_stream_aes128ctr_portable_KEYBYTES -#define crypto_stream_aes128ctr_NONCEBYTES crypto_stream_aes128ctr_portable_NONCEBYTES -#define crypto_stream_aes128ctr_BEFORENMBYTES crypto_stream_aes128ctr_portable_BEFORENMBYTES -#define crypto_stream_aes128ctr_IMPLEMENTATION "crypto_stream/aes128ctr/portable" -#ifndef crypto_stream_aes128ctr_portable_VERSION -#define crypto_stream_aes128ctr_portable_VERSION "-" -#endif -#define crypto_stream_aes128ctr_VERSION crypto_stream_aes128ctr_portable_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h deleted file mode 100644 index c96d20b4..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_salsa20_H -#define crypto_stream_salsa20_H - -#define crypto_stream_salsa20_amd64_xmm6_KEYBYTES 32 -#define crypto_stream_salsa20_amd64_xmm6_NONCEBYTES 8 -#ifdef __cplusplus -#include -extern std::string crypto_stream_salsa20_amd64_xmm6(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_salsa20_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_salsa20_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_salsa20 crypto_stream_salsa20_amd64_xmm6 -#define crypto_stream_salsa20_xor crypto_stream_salsa20_amd64_xmm6_xor -#define crypto_stream_salsa20_beforenm crypto_stream_salsa20_amd64_xmm6_beforenm -#define crypto_stream_salsa20_afternm crypto_stream_salsa20_amd64_xmm6_afternm -#define crypto_stream_salsa20_xor_afternm crypto_stream_salsa20_amd64_xmm6_xor_afternm -#define crypto_stream_salsa20_KEYBYTES crypto_stream_salsa20_amd64_xmm6_KEYBYTES -#define crypto_stream_salsa20_NONCEBYTES crypto_stream_salsa20_amd64_xmm6_NONCEBYTES -#define crypto_stream_salsa20_BEFORENMBYTES crypto_stream_salsa20_amd64_xmm6_BEFORENMBYTES -#define crypto_stream_salsa20_IMPLEMENTATION "crypto_stream/salsa20/amd64_xmm6" -#ifndef crypto_stream_salsa20_amd64_xmm6_VERSION -#define crypto_stream_salsa20_amd64_xmm6_VERSION "-" -#endif -#define crypto_stream_salsa20_VERSION crypto_stream_salsa20_amd64_xmm6_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h deleted file mode 100644 index 051e4e39..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_salsa2012_H -#define crypto_stream_salsa2012_H - -#define crypto_stream_salsa2012_amd64_xmm6_KEYBYTES 32 -#define crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES 8 -#ifdef __cplusplus -#include -extern std::string crypto_stream_salsa2012_amd64_xmm6(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_salsa2012_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_salsa2012 crypto_stream_salsa2012_amd64_xmm6 -#define crypto_stream_salsa2012_xor crypto_stream_salsa2012_amd64_xmm6_xor -#define crypto_stream_salsa2012_beforenm crypto_stream_salsa2012_amd64_xmm6_beforenm -#define crypto_stream_salsa2012_afternm crypto_stream_salsa2012_amd64_xmm6_afternm -#define crypto_stream_salsa2012_xor_afternm crypto_stream_salsa2012_amd64_xmm6_xor_afternm -#define crypto_stream_salsa2012_KEYBYTES crypto_stream_salsa2012_amd64_xmm6_KEYBYTES -#define crypto_stream_salsa2012_NONCEBYTES crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES -#define crypto_stream_salsa2012_BEFORENMBYTES crypto_stream_salsa2012_amd64_xmm6_BEFORENMBYTES -#define crypto_stream_salsa2012_IMPLEMENTATION "crypto_stream/salsa2012/amd64_xmm6" -#ifndef crypto_stream_salsa2012_amd64_xmm6_VERSION -#define crypto_stream_salsa2012_amd64_xmm6_VERSION "-" -#endif -#define crypto_stream_salsa2012_VERSION crypto_stream_salsa2012_amd64_xmm6_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h deleted file mode 100644 index 4bd470c3..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_salsa208_H -#define crypto_stream_salsa208_H - -#define crypto_stream_salsa208_amd64_xmm6_KEYBYTES 32 -#define crypto_stream_salsa208_amd64_xmm6_NONCEBYTES 8 -#ifdef __cplusplus -#include -extern std::string crypto_stream_salsa208_amd64_xmm6(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_salsa208_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_salsa208_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_salsa208 crypto_stream_salsa208_amd64_xmm6 -#define crypto_stream_salsa208_xor crypto_stream_salsa208_amd64_xmm6_xor -#define crypto_stream_salsa208_beforenm crypto_stream_salsa208_amd64_xmm6_beforenm -#define crypto_stream_salsa208_afternm crypto_stream_salsa208_amd64_xmm6_afternm -#define crypto_stream_salsa208_xor_afternm crypto_stream_salsa208_amd64_xmm6_xor_afternm -#define crypto_stream_salsa208_KEYBYTES crypto_stream_salsa208_amd64_xmm6_KEYBYTES -#define crypto_stream_salsa208_NONCEBYTES crypto_stream_salsa208_amd64_xmm6_NONCEBYTES -#define crypto_stream_salsa208_BEFORENMBYTES crypto_stream_salsa208_amd64_xmm6_BEFORENMBYTES -#define crypto_stream_salsa208_IMPLEMENTATION "crypto_stream/salsa208/amd64_xmm6" -#ifndef crypto_stream_salsa208_amd64_xmm6_VERSION -#define crypto_stream_salsa208_amd64_xmm6_VERSION "-" -#endif -#define crypto_stream_salsa208_VERSION crypto_stream_salsa208_amd64_xmm6_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h deleted file mode 100644 index d75268c6..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_xsalsa20_H -#define crypto_stream_xsalsa20_H - -#define crypto_stream_xsalsa20_ref_KEYBYTES 32 -#define crypto_stream_xsalsa20_ref_NONCEBYTES 24 -#ifdef __cplusplus -#include -extern std::string crypto_stream_xsalsa20_ref(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_xsalsa20_ref_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_xsalsa20_ref(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_xsalsa20 crypto_stream_xsalsa20_ref -#define crypto_stream_xsalsa20_xor crypto_stream_xsalsa20_ref_xor -#define crypto_stream_xsalsa20_beforenm crypto_stream_xsalsa20_ref_beforenm -#define crypto_stream_xsalsa20_afternm crypto_stream_xsalsa20_ref_afternm -#define crypto_stream_xsalsa20_xor_afternm crypto_stream_xsalsa20_ref_xor_afternm -#define crypto_stream_xsalsa20_KEYBYTES crypto_stream_xsalsa20_ref_KEYBYTES -#define crypto_stream_xsalsa20_NONCEBYTES crypto_stream_xsalsa20_ref_NONCEBYTES -#define crypto_stream_xsalsa20_BEFORENMBYTES crypto_stream_xsalsa20_ref_BEFORENMBYTES -#define crypto_stream_xsalsa20_IMPLEMENTATION "crypto_stream/xsalsa20/ref" -#ifndef crypto_stream_xsalsa20_ref_VERSION -#define crypto_stream_xsalsa20_ref_VERSION "-" -#endif -#define crypto_stream_xsalsa20_VERSION crypto_stream_xsalsa20_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h deleted file mode 100644 index b0ce9656..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef crypto_types_h -#define crypto_types_h -typedef short crypto_int16; -typedef int crypto_int32; -typedef long long crypto_int64; -typedef signed char crypto_int8; -typedef unsigned short crypto_uint16; -typedef unsigned int crypto_uint32; -typedef unsigned long long crypto_uint64; -typedef unsigned char crypto_uint8; -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h deleted file mode 100644 index 6bf6ca11..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef crypto_verify_16_H -#define crypto_verify_16_H - -#define crypto_verify_16_ref_BYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_verify_16_ref(const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_verify_16 crypto_verify_16_ref -#define crypto_verify_16_BYTES crypto_verify_16_ref_BYTES -#define crypto_verify_16_IMPLEMENTATION "crypto_verify/16/ref" -#ifndef crypto_verify_16_ref_VERSION -#define crypto_verify_16_ref_VERSION "-" -#endif -#define crypto_verify_16_VERSION crypto_verify_16_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h deleted file mode 100644 index bd5fc644..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef crypto_verify_32_H -#define crypto_verify_32_H - -#define crypto_verify_32_ref_BYTES 32 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_verify_32_ref(const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_verify_32 crypto_verify_32_ref -#define crypto_verify_32_BYTES crypto_verify_32_ref_BYTES -#define crypto_verify_32_IMPLEMENTATION "crypto_verify/32/ref" -#ifndef crypto_verify_32_ref_VERSION -#define crypto_verify_32_ref_VERSION "-" -#endif -#define crypto_verify_32_VERSION crypto_verify_32_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/libnacl.a b/ext/bin/cnacl-osx-amd64/libnacl.a deleted file mode 100644 index d46fffab..00000000 Binary files a/ext/bin/cnacl-osx-amd64/libnacl.a and /dev/null differ diff --git a/make-mac.mk b/make-mac.mk index b71ca2fe..8ff1b772 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,12 +33,6 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif -# Use precompiled extremely fast Salsa20/12 from "cnacl" included in ext/bin -# See https://github.com/cjdelisle/cnacl -DEFS+=-DZT_USE_LIBSODIUM -CFLAGS+=-Iext/bin/cnacl-osx-amd64/include -LIBS+=ext/bin/cnacl-osx-amd64/libnacl.a - ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif diff --git a/node/Node.cpp b/node/Node.cpp index 55fb4e72..2b3f7996 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -50,7 +50,6 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 _RR(this), RR(&_RR), _uPtr(uptr), - _prngStreamPtr(0), _now(now), _lastPingCheck(0), _lastHousekeepingRun(0) @@ -59,19 +58,14 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 throw std::runtime_error("callbacks struct version mismatch"); memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); + Utils::getSecureRandom((void *)_prngState,sizeof(_prngState)); + _online = false; memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); - // Use Salsa20 alone as a high-quality non-crypto PRNG - char foo[64]; - Utils::getSecureRandom(foo,64); - _prng.init(foo,foo + 32); - memset(_prngStream,0,sizeof(_prngStream)); - _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - std::string idtmp(dataStoreGet(tptr,"identity.secret")); if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { TRACE("identity.secret not found, generating..."); @@ -701,10 +695,14 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) uint64_t Node::prng() { - unsigned int p = (++_prngStreamPtr % ZT_NODE_PRNG_BUF_SIZE); - if (!p) - _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - return _prngStream[p]; + // https://en.wikipedia.org/wiki/Xorshift#xorshift.2B + uint64_t x = _prngState[0]; + const uint64_t y = _prngState[1]; + _prngState[0] = y; + x ^= x << 23; + const uint64_t z = x ^ y ^ (x >> 17) ^ (y >> 26); + _prngState[1] = z; + return z + y; } void Node::postCircuitTestReport(const ZT_CircuitTestReport *report) diff --git a/node/Node.hpp b/node/Node.hpp index 03bd7a8c..d25a619b 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -50,9 +50,6 @@ #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 #define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 -// Size of PRNG stream buffer -#define ZT_NODE_PRNG_BUF_SIZE 64 - namespace ZeroTier { class World; @@ -312,13 +309,10 @@ private: Mutex _backgroundTasksLock; - unsigned int _prngStreamPtr; - Salsa20 _prng; - uint64_t _prngStream[ZT_NODE_PRNG_BUF_SIZE]; // repeatedly encrypted with _prng to yield a high-quality non-crypto PRNG stream - uint64_t _now; uint64_t _lastPingCheck; uint64_t _lastHousekeepingRun; + volatile uint64_t _prngState[2]; bool _online; }; diff --git a/node/Salsa20.cpp b/node/Salsa20.cpp index 1e2b4b0f..2a802555 100644 --- a/node/Salsa20.cpp +++ b/node/Salsa20.cpp @@ -10,8 +10,6 @@ #include "Constants.hpp" #include "Salsa20.hpp" -#ifndef ZT_USE_LIBSODIUM - #define ROTATE(v,c) (((v) << (c)) | ((v) >> (32 - (c)))) #define XOR(v,w) ((v) ^ (w)) #define PLUS(v,w) ((uint32_t)((v) + (w))) @@ -1345,5 +1343,3 @@ void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) } } // namespace ZeroTier - -#endif // !ZT_USE_LIBSODIUM diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 5e4c68be..c6af5700 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -15,77 +15,6 @@ #include "Constants.hpp" #include "Utils.hpp" -#ifdef ZT_USE_LIBSODIUM - -#include -#include - -namespace ZeroTier { - -/** - * Salsa20 stream cipher - */ -class Salsa20 -{ -public: - Salsa20() {} - ~Salsa20() { Utils::burn(_k,sizeof(_k)); } - - /** - * @param key 256-bit (32 byte) key - * @param iv 64-bit initialization vector - */ - Salsa20(const void *key,const void *iv) - { - memcpy(_k,key,32); - memcpy(&_iv,iv,8); - } - - /** - * Initialize cipher - * - * @param key Key bits - * @param iv 64-bit initialization vector - */ - inline void init(const void *key,const void *iv) - { - memcpy(_k,key,32); - memcpy(&_iv,iv,8); - } - - /** - * Encrypt/decrypt data using Salsa20/12 - * - * @param in Input data - * @param out Output buffer - * @param bytes Length of data - */ - inline void crypt12(const void *in,void *out,unsigned int bytes) - { - crypto_stream_salsa2012_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); - } - - /** - * Encrypt/decrypt data using Salsa20/20 - * - * @param in Input data - * @param out Output buffer - * @param bytes Length of data - */ - inline void crypt20(const void *in,void *out,unsigned int bytes) - { - crypto_stream_salsa20_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); - } - -private: - uint64_t _k[4]; - uint64_t _iv; -}; - -} // namespace ZeroTier - -#else // !ZT_USE_LIBSODIUM - #if (!defined(ZT_SALSA20_SSE)) && (defined(__SSE2__) || defined(__WINDOWS__)) #define ZT_SALSA20_SSE 1 #endif @@ -105,6 +34,11 @@ public: Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } + /** + * If this returns true, crypt can only be done once + */ + static inline bool singleUseOnly() { return false; } + /** * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector @@ -151,6 +85,4 @@ private: } // namespace ZeroTier -#endif // ZT_USE_LIBSODIUM - #endif diff --git a/node/Utils.cpp b/node/Utils.cpp index 92d14d19..9ce1bf05 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -177,6 +177,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } randomPtr = 0; s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); + s20.init(randomBuf,randomBuf); } ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } @@ -209,6 +210,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } randomPtr = 0; s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); + s20.init(randomBuf,randomBuf); } ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } -- cgit v1.2.3 From 4938e82795ffa0bebeb5921df84bd3e362ba2f46 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 18:01:51 -0700 Subject: Delete junk. --- node/Salsa20.hpp | 5 ----- 1 file changed, 5 deletions(-) (limited to 'node') diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index c6af5700..448c8f1a 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -34,11 +34,6 @@ public: Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } - /** - * If this returns true, crypt can only be done once - */ - static inline bool singleUseOnly() { return false; } - /** * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector -- cgit v1.2.3 From a1e94154bebe17a24d2eed43be7d866e93c061fe Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Apr 2017 08:45:37 -0700 Subject: Just incorporate the X64 ASM version of Salsa20/12 for X64 platforms. This gives us (for example) 1.5gb/sec encryption on a Core i5 2.8ghz. --- ext/x64-salsa2012-asm/README.md | 6 + ext/x64-salsa2012-asm/salsa2012.h | 13 + ext/x64-salsa2012-asm/salsa2012.s | 4488 +++++++++++++++++++++++++++++++++++++ make-mac.mk | 7 + node/Packet.cpp | 65 +- selftest.cpp | 22 + 6 files changed, 4597 insertions(+), 4 deletions(-) create mode 100644 ext/x64-salsa2012-asm/README.md create mode 100644 ext/x64-salsa2012-asm/salsa2012.h create mode 100644 ext/x64-salsa2012-asm/salsa2012.s (limited to 'node') diff --git a/ext/x64-salsa2012-asm/README.md b/ext/x64-salsa2012-asm/README.md new file mode 100644 index 00000000..a69a1a67 --- /dev/null +++ b/ext/x64-salsa2012-asm/README.md @@ -0,0 +1,6 @@ +Blazingly fast X64 ASM implementation of Salsa20/12 +====== + +This is ripped from the [cnacl](https://github.com/cjdelisle/cnacl) source. The actual code is by Danial J. Bernstein and is in the public domain. + +This is included on Linux and Mac 64-bit builds and is significantly faster than the SSE intrinsics or C versions. It's used for packet encode/decode only since its use differs a bit from the regular Salsa20 C++ class. Specifically it lacks the ability to be called on multiple blocks, preferring instead to take a key and a single stream to encrypt and that's it. diff --git a/ext/x64-salsa2012-asm/salsa2012.h b/ext/x64-salsa2012-asm/salsa2012.h new file mode 100644 index 00000000..d47059b4 --- /dev/null +++ b/ext/x64-salsa2012-asm/salsa2012.h @@ -0,0 +1,13 @@ +#ifdef __cplusplus +extern "C" { +#endif + +// output, outlen, nonce, key (256-bit / 32-byte) +extern int zt_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); + +// ciphertext, message, mlen, nonce, key +extern int zt_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); + +#ifdef __cplusplus +} +#endif diff --git a/ext/x64-salsa2012-asm/salsa2012.s b/ext/x64-salsa2012-asm/salsa2012.s new file mode 100644 index 00000000..699c89ac --- /dev/null +++ b/ext/x64-salsa2012-asm/salsa2012.s @@ -0,0 +1,4488 @@ +# qhasm: enter zt_salsa2012_amd64_xmm6 +.text +.p2align 5 +.globl _zt_salsa2012_amd64_xmm6 +.globl zt_salsa2012_amd64_xmm6 +_zt_salsa2012_amd64_xmm6: +zt_salsa2012_amd64_xmm6: +mov %rsp,%r11 +and $31,%r11 +add $480,%r11 +sub %r11,%rsp + +# qhasm: r11_stack = r11_caller +# asm 1: movq r11_stack=stack64#1 +# asm 2: movq r11_stack=352(%rsp) +movq %r11,352(%rsp) + +# qhasm: r12_stack = r12_caller +# asm 1: movq r12_stack=stack64#2 +# asm 2: movq r12_stack=360(%rsp) +movq %r12,360(%rsp) + +# qhasm: r13_stack = r13_caller +# asm 1: movq r13_stack=stack64#3 +# asm 2: movq r13_stack=368(%rsp) +movq %r13,368(%rsp) + +# qhasm: r14_stack = r14_caller +# asm 1: movq r14_stack=stack64#4 +# asm 2: movq r14_stack=376(%rsp) +movq %r14,376(%rsp) + +# qhasm: r15_stack = r15_caller +# asm 1: movq r15_stack=stack64#5 +# asm 2: movq r15_stack=384(%rsp) +movq %r15,384(%rsp) + +# qhasm: rbx_stack = rbx_caller +# asm 1: movq rbx_stack=stack64#6 +# asm 2: movq rbx_stack=392(%rsp) +movq %rbx,392(%rsp) + +# qhasm: rbp_stack = rbp_caller +# asm 1: movq rbp_stack=stack64#7 +# asm 2: movq rbp_stack=400(%rsp) +movq %rbp,400(%rsp) + +# qhasm: bytes = arg2 +# asm 1: mov bytes=int64#6 +# asm 2: mov bytes=%r9 +mov %rsi,%r9 + +# qhasm: out = arg1 +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdi,%rdi + +# qhasm: m = out +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rdi,%rsi + +# qhasm: iv = arg3 +# asm 1: mov iv=int64#3 +# asm 2: mov iv=%rdx +mov %rdx,%rdx + +# qhasm: k = arg4 +# asm 1: mov k=int64#8 +# asm 2: mov k=%r10 +mov %rcx,%r10 + +# qhasm: unsigned>? bytes - 0 +# asm 1: cmp $0, +jbe ._done + +# qhasm: a = 0 +# asm 1: mov $0,>a=int64#7 +# asm 2: mov $0,>a=%rax +mov $0,%rax + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = a; --i } +rep stosb + +# qhasm: out -= bytes +# asm 1: sub r11_stack=stack64#1 +# asm 2: movq r11_stack=352(%rsp) +movq %r11,352(%rsp) + +# qhasm: r12_stack = r12_caller +# asm 1: movq r12_stack=stack64#2 +# asm 2: movq r12_stack=360(%rsp) +movq %r12,360(%rsp) + +# qhasm: r13_stack = r13_caller +# asm 1: movq r13_stack=stack64#3 +# asm 2: movq r13_stack=368(%rsp) +movq %r13,368(%rsp) + +# qhasm: r14_stack = r14_caller +# asm 1: movq r14_stack=stack64#4 +# asm 2: movq r14_stack=376(%rsp) +movq %r14,376(%rsp) + +# qhasm: r15_stack = r15_caller +# asm 1: movq r15_stack=stack64#5 +# asm 2: movq r15_stack=384(%rsp) +movq %r15,384(%rsp) + +# qhasm: rbx_stack = rbx_caller +# asm 1: movq rbx_stack=stack64#6 +# asm 2: movq rbx_stack=392(%rsp) +movq %rbx,392(%rsp) + +# qhasm: rbp_stack = rbp_caller +# asm 1: movq rbp_stack=stack64#7 +# asm 2: movq rbp_stack=400(%rsp) +movq %rbp,400(%rsp) + +# qhasm: out = arg1 +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdi,%rdi + +# qhasm: m = arg2 +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rsi,%rsi + +# qhasm: bytes = arg3 +# asm 1: mov bytes=int64#6 +# asm 2: mov bytes=%r9 +mov %rdx,%r9 + +# qhasm: iv = arg4 +# asm 1: mov iv=int64#3 +# asm 2: mov iv=%rdx +mov %rcx,%rdx + +# qhasm: k = arg5 +# asm 1: mov k=int64#8 +# asm 2: mov k=%r10 +mov %r8,%r10 + +# qhasm: unsigned>? bytes - 0 +# asm 1: cmp $0, +jbe ._done +# comment:fp stack unchanged by fallthrough + +# qhasm: start: +._start: + +# qhasm: in12 = *(uint32 *) (k + 20) +# asm 1: movl 20(in12=int64#4d +# asm 2: movl 20(in12=%ecx +movl 20(%r10),%ecx + +# qhasm: in1 = *(uint32 *) (k + 0) +# asm 1: movl 0(in1=int64#5d +# asm 2: movl 0(in1=%r8d +movl 0(%r10),%r8d + +# qhasm: in6 = *(uint32 *) (iv + 0) +# asm 1: movl 0(in6=int64#7d +# asm 2: movl 0(in6=%eax +movl 0(%rdx),%eax + +# qhasm: in11 = *(uint32 *) (k + 16) +# asm 1: movl 16(in11=int64#9d +# asm 2: movl 16(in11=%r11d +movl 16(%r10),%r11d + +# qhasm: ((uint32 *)&x1)[0] = in12 +# asm 1: movl x1=stack128#1 +# asm 2: movl x1=0(%rsp) +movl %ecx,0(%rsp) + +# qhasm: ((uint32 *)&x1)[1] = in1 +# asm 1: movl in8=int64#4 +# asm 2: mov $0,>in8=%rcx +mov $0,%rcx + +# qhasm: in13 = *(uint32 *) (k + 24) +# asm 1: movl 24(in13=int64#5d +# asm 2: movl 24(in13=%r8d +movl 24(%r10),%r8d + +# qhasm: in2 = *(uint32 *) (k + 4) +# asm 1: movl 4(in2=int64#7d +# asm 2: movl 4(in2=%eax +movl 4(%r10),%eax + +# qhasm: in7 = *(uint32 *) (iv + 4) +# asm 1: movl 4(in7=int64#3d +# asm 2: movl 4(in7=%edx +movl 4(%rdx),%edx + +# qhasm: ((uint32 *)&x2)[0] = in8 +# asm 1: movl x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %ecx,16(%rsp) + +# qhasm: ((uint32 *)&x2)[1] = in13 +# asm 1: movl in4=int64#3d +# asm 2: movl 12(in4=%edx +movl 12(%r10),%edx + +# qhasm: in9 = 0 +# asm 1: mov $0,>in9=int64#4 +# asm 2: mov $0,>in9=%rcx +mov $0,%rcx + +# qhasm: in14 = *(uint32 *) (k + 28) +# asm 1: movl 28(in14=int64#5d +# asm 2: movl 28(in14=%r8d +movl 28(%r10),%r8d + +# qhasm: in3 = *(uint32 *) (k + 8) +# asm 1: movl 8(in3=int64#7d +# asm 2: movl 8(in3=%eax +movl 8(%r10),%eax + +# qhasm: ((uint32 *)&x3)[0] = in4 +# asm 1: movl x3=stack128#3 +# asm 2: movl x3=32(%rsp) +movl %edx,32(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl in0=int64#3 +# asm 2: mov $1634760805,>in0=%rdx +mov $1634760805,%rdx + +# qhasm: in5 = 857760878 +# asm 1: mov $857760878,>in5=int64#4 +# asm 2: mov $857760878,>in5=%rcx +mov $857760878,%rcx + +# qhasm: in10 = 2036477234 +# asm 1: mov $2036477234,>in10=int64#5 +# asm 2: mov $2036477234,>in10=%r8 +mov $2036477234,%r8 + +# qhasm: in15 = 1797285236 +# asm 1: mov $1797285236,>in15=int64#7 +# asm 2: mov $1797285236,>in15=%rax +mov $1797285236,%rax + +# qhasm: ((uint32 *)&x0)[0] = in0 +# asm 1: movl x0=stack128#4 +# asm 2: movl x0=48(%rsp) +movl %edx,48(%rsp) + +# qhasm: ((uint32 *)&x0)[1] = in5 +# asm 1: movl z0=int6464#1 +# asm 2: movdqa z0=%xmm0 +movdqa 48(%rsp),%xmm0 + +# qhasm: z5 = z0[1,1,1,1] +# asm 1: pshufd $0x55,z5=int6464#2 +# asm 2: pshufd $0x55,z5=%xmm1 +pshufd $0x55,%xmm0,%xmm1 + +# qhasm: z10 = z0[2,2,2,2] +# asm 1: pshufd $0xaa,z10=int6464#3 +# asm 2: pshufd $0xaa,z10=%xmm2 +pshufd $0xaa,%xmm0,%xmm2 + +# qhasm: z15 = z0[3,3,3,3] +# asm 1: pshufd $0xff,z15=int6464#4 +# asm 2: pshufd $0xff,z15=%xmm3 +pshufd $0xff,%xmm0,%xmm3 + +# qhasm: z0 = z0[0,0,0,0] +# asm 1: pshufd $0x00,z0=int6464#1 +# asm 2: pshufd $0x00,z0=%xmm0 +pshufd $0x00,%xmm0,%xmm0 + +# qhasm: orig5 = z5 +# asm 1: movdqa orig5=stack128#5 +# asm 2: movdqa orig5=64(%rsp) +movdqa %xmm1,64(%rsp) + +# qhasm: orig10 = z10 +# asm 1: movdqa orig10=stack128#6 +# asm 2: movdqa orig10=80(%rsp) +movdqa %xmm2,80(%rsp) + +# qhasm: orig15 = z15 +# asm 1: movdqa orig15=stack128#7 +# asm 2: movdqa orig15=96(%rsp) +movdqa %xmm3,96(%rsp) + +# qhasm: orig0 = z0 +# asm 1: movdqa orig0=stack128#8 +# asm 2: movdqa orig0=112(%rsp) +movdqa %xmm0,112(%rsp) + +# qhasm: z1 = x1 +# asm 1: movdqa z1=int6464#1 +# asm 2: movdqa z1=%xmm0 +movdqa 0(%rsp),%xmm0 + +# qhasm: z6 = z1[2,2,2,2] +# asm 1: pshufd $0xaa,z6=int6464#2 +# asm 2: pshufd $0xaa,z6=%xmm1 +pshufd $0xaa,%xmm0,%xmm1 + +# qhasm: z11 = z1[3,3,3,3] +# asm 1: pshufd $0xff,z11=int6464#3 +# asm 2: pshufd $0xff,z11=%xmm2 +pshufd $0xff,%xmm0,%xmm2 + +# qhasm: z12 = z1[0,0,0,0] +# asm 1: pshufd $0x00,z12=int6464#4 +# asm 2: pshufd $0x00,z12=%xmm3 +pshufd $0x00,%xmm0,%xmm3 + +# qhasm: z1 = z1[1,1,1,1] +# asm 1: pshufd $0x55,z1=int6464#1 +# asm 2: pshufd $0x55,z1=%xmm0 +pshufd $0x55,%xmm0,%xmm0 + +# qhasm: orig6 = z6 +# asm 1: movdqa orig6=stack128#9 +# asm 2: movdqa orig6=128(%rsp) +movdqa %xmm1,128(%rsp) + +# qhasm: orig11 = z11 +# asm 1: movdqa orig11=stack128#10 +# asm 2: movdqa orig11=144(%rsp) +movdqa %xmm2,144(%rsp) + +# qhasm: orig12 = z12 +# asm 1: movdqa orig12=stack128#11 +# asm 2: movdqa orig12=160(%rsp) +movdqa %xmm3,160(%rsp) + +# qhasm: orig1 = z1 +# asm 1: movdqa orig1=stack128#12 +# asm 2: movdqa orig1=176(%rsp) +movdqa %xmm0,176(%rsp) + +# qhasm: z2 = x2 +# asm 1: movdqa z2=int6464#1 +# asm 2: movdqa z2=%xmm0 +movdqa 16(%rsp),%xmm0 + +# qhasm: z7 = z2[3,3,3,3] +# asm 1: pshufd $0xff,z7=int6464#2 +# asm 2: pshufd $0xff,z7=%xmm1 +pshufd $0xff,%xmm0,%xmm1 + +# qhasm: z13 = z2[1,1,1,1] +# asm 1: pshufd $0x55,z13=int6464#3 +# asm 2: pshufd $0x55,z13=%xmm2 +pshufd $0x55,%xmm0,%xmm2 + +# qhasm: z2 = z2[2,2,2,2] +# asm 1: pshufd $0xaa,z2=int6464#1 +# asm 2: pshufd $0xaa,z2=%xmm0 +pshufd $0xaa,%xmm0,%xmm0 + +# qhasm: orig7 = z7 +# asm 1: movdqa orig7=stack128#13 +# asm 2: movdqa orig7=192(%rsp) +movdqa %xmm1,192(%rsp) + +# qhasm: orig13 = z13 +# asm 1: movdqa orig13=stack128#14 +# asm 2: movdqa orig13=208(%rsp) +movdqa %xmm2,208(%rsp) + +# qhasm: orig2 = z2 +# asm 1: movdqa orig2=stack128#15 +# asm 2: movdqa orig2=224(%rsp) +movdqa %xmm0,224(%rsp) + +# qhasm: z3 = x3 +# asm 1: movdqa z3=int6464#1 +# asm 2: movdqa z3=%xmm0 +movdqa 32(%rsp),%xmm0 + +# qhasm: z4 = z3[0,0,0,0] +# asm 1: pshufd $0x00,z4=int6464#2 +# asm 2: pshufd $0x00,z4=%xmm1 +pshufd $0x00,%xmm0,%xmm1 + +# qhasm: z14 = z3[2,2,2,2] +# asm 1: pshufd $0xaa,z14=int6464#3 +# asm 2: pshufd $0xaa,z14=%xmm2 +pshufd $0xaa,%xmm0,%xmm2 + +# qhasm: z3 = z3[3,3,3,3] +# asm 1: pshufd $0xff,z3=int6464#1 +# asm 2: pshufd $0xff,z3=%xmm0 +pshufd $0xff,%xmm0,%xmm0 + +# qhasm: orig4 = z4 +# asm 1: movdqa orig4=stack128#16 +# asm 2: movdqa orig4=240(%rsp) +movdqa %xmm1,240(%rsp) + +# qhasm: orig14 = z14 +# asm 1: movdqa orig14=stack128#17 +# asm 2: movdqa orig14=256(%rsp) +movdqa %xmm2,256(%rsp) + +# qhasm: orig3 = z3 +# asm 1: movdqa orig3=stack128#18 +# asm 2: movdqa orig3=272(%rsp) +movdqa %xmm0,272(%rsp) + +# qhasm: bytesatleast256: +._bytesatleast256: + +# qhasm: in8 = ((uint32 *)&x2)[0] +# asm 1: movl in8=int64#3d +# asm 2: movl in8=%edx +movl 16(%rsp),%edx + +# qhasm: in9 = ((uint32 *)&x3)[1] +# asm 1: movl 4+in9=int64#4d +# asm 2: movl 4+in9=%ecx +movl 4+32(%rsp),%ecx + +# qhasm: ((uint32 *) &orig8)[0] = in8 +# asm 1: movl orig8=stack128#19 +# asm 2: movl orig8=288(%rsp) +movl %edx,288(%rsp) + +# qhasm: ((uint32 *) &orig9)[0] = in9 +# asm 1: movl orig9=stack128#20 +# asm 2: movl orig9=304(%rsp) +movl %ecx,304(%rsp) + +# qhasm: in8 += 1 +# asm 1: add $1,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %edx,16(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl bytes_backup=stack64#8 +# asm 2: movq bytes_backup=408(%rsp) +movq %r9,408(%rsp) + +# qhasm: i = 12 +# asm 1: mov $12,>i=int64#3 +# asm 2: mov $12,>i=%rdx +mov $12,%rdx + +# qhasm: z5 = orig5 +# asm 1: movdqa z5=int6464#1 +# asm 2: movdqa z5=%xmm0 +movdqa 64(%rsp),%xmm0 + +# qhasm: z10 = orig10 +# asm 1: movdqa z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 80(%rsp),%xmm1 + +# qhasm: z15 = orig15 +# asm 1: movdqa z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 96(%rsp),%xmm2 + +# qhasm: z14 = orig14 +# asm 1: movdqa z14=int6464#4 +# asm 2: movdqa z14=%xmm3 +movdqa 256(%rsp),%xmm3 + +# qhasm: z3 = orig3 +# asm 1: movdqa z3=int6464#5 +# asm 2: movdqa z3=%xmm4 +movdqa 272(%rsp),%xmm4 + +# qhasm: z6 = orig6 +# asm 1: movdqa z6=int6464#6 +# asm 2: movdqa z6=%xmm5 +movdqa 128(%rsp),%xmm5 + +# qhasm: z11 = orig11 +# asm 1: movdqa z11=int6464#7 +# asm 2: movdqa z11=%xmm6 +movdqa 144(%rsp),%xmm6 + +# qhasm: z1 = orig1 +# asm 1: movdqa z1=int6464#8 +# asm 2: movdqa z1=%xmm7 +movdqa 176(%rsp),%xmm7 + +# qhasm: z7 = orig7 +# asm 1: movdqa z7=int6464#9 +# asm 2: movdqa z7=%xmm8 +movdqa 192(%rsp),%xmm8 + +# qhasm: z13 = orig13 +# asm 1: movdqa z13=int6464#10 +# asm 2: movdqa z13=%xmm9 +movdqa 208(%rsp),%xmm9 + +# qhasm: z2 = orig2 +# asm 1: movdqa z2=int6464#11 +# asm 2: movdqa z2=%xmm10 +movdqa 224(%rsp),%xmm10 + +# qhasm: z9 = orig9 +# asm 1: movdqa z9=int6464#12 +# asm 2: movdqa z9=%xmm11 +movdqa 304(%rsp),%xmm11 + +# qhasm: z0 = orig0 +# asm 1: movdqa z0=int6464#13 +# asm 2: movdqa z0=%xmm12 +movdqa 112(%rsp),%xmm12 + +# qhasm: z12 = orig12 +# asm 1: movdqa z12=int6464#14 +# asm 2: movdqa z12=%xmm13 +movdqa 160(%rsp),%xmm13 + +# qhasm: z4 = orig4 +# asm 1: movdqa z4=int6464#15 +# asm 2: movdqa z4=%xmm14 +movdqa 240(%rsp),%xmm14 + +# qhasm: z8 = orig8 +# asm 1: movdqa z8=int6464#16 +# asm 2: movdqa z8=%xmm15 +movdqa 288(%rsp),%xmm15 + +# qhasm: mainloop1: +._mainloop1: + +# qhasm: z10_stack = z10 +# asm 1: movdqa z10_stack=stack128#21 +# asm 2: movdqa z10_stack=320(%rsp) +movdqa %xmm1,320(%rsp) + +# qhasm: z15_stack = z15 +# asm 1: movdqa z15_stack=stack128#22 +# asm 2: movdqa z15_stack=336(%rsp) +movdqa %xmm2,336(%rsp) + +# qhasm: y4 = z12 +# asm 1: movdqa y4=int6464#2 +# asm 2: movdqa y4=%xmm1 +movdqa %xmm13,%xmm1 + +# qhasm: uint32323232 y4 += z0 +# asm 1: paddd r4=int6464#3 +# asm 2: movdqa r4=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y9=int6464#2 +# asm 2: movdqa y9=%xmm1 +movdqa %xmm7,%xmm1 + +# qhasm: uint32323232 y9 += z5 +# asm 1: paddd r9=int6464#3 +# asm 2: movdqa r9=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y9 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y8=int6464#2 +# asm 2: movdqa y8=%xmm1 +movdqa %xmm12,%xmm1 + +# qhasm: uint32323232 y8 += z4 +# asm 1: paddd r8=int6464#3 +# asm 2: movdqa r8=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y8 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y13=int6464#2 +# asm 2: movdqa y13=%xmm1 +movdqa %xmm0,%xmm1 + +# qhasm: uint32323232 y13 += z9 +# asm 1: paddd r13=int6464#3 +# asm 2: movdqa r13=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y13 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y12=int6464#2 +# asm 2: movdqa y12=%xmm1 +movdqa %xmm14,%xmm1 + +# qhasm: uint32323232 y12 += z8 +# asm 1: paddd r12=int6464#3 +# asm 2: movdqa r12=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y12 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y1=int6464#2 +# asm 2: movdqa y1=%xmm1 +movdqa %xmm11,%xmm1 + +# qhasm: uint32323232 y1 += z13 +# asm 1: paddd r1=int6464#3 +# asm 2: movdqa r1=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y1 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y0=int6464#2 +# asm 2: movdqa y0=%xmm1 +movdqa %xmm15,%xmm1 + +# qhasm: uint32323232 y0 += z12 +# asm 1: paddd r0=int6464#3 +# asm 2: movdqa r0=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y0 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 320(%rsp),%xmm1 + +# qhasm: z0_stack = z0 +# asm 1: movdqa z0_stack=stack128#21 +# asm 2: movdqa z0_stack=320(%rsp) +movdqa %xmm12,320(%rsp) + +# qhasm: y5 = z13 +# asm 1: movdqa y5=int6464#3 +# asm 2: movdqa y5=%xmm2 +movdqa %xmm9,%xmm2 + +# qhasm: uint32323232 y5 += z1 +# asm 1: paddd r5=int6464#13 +# asm 2: movdqa r5=%xmm12 +movdqa %xmm2,%xmm12 + +# qhasm: uint32323232 y5 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y14=int6464#3 +# asm 2: movdqa y14=%xmm2 +movdqa %xmm5,%xmm2 + +# qhasm: uint32323232 y14 += z10 +# asm 1: paddd r14=int6464#13 +# asm 2: movdqa r14=%xmm12 +movdqa %xmm2,%xmm12 + +# qhasm: uint32323232 y14 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 336(%rsp),%xmm2 + +# qhasm: z5_stack = z5 +# asm 1: movdqa z5_stack=stack128#22 +# asm 2: movdqa z5_stack=336(%rsp) +movdqa %xmm0,336(%rsp) + +# qhasm: y3 = z11 +# asm 1: movdqa y3=int6464#1 +# asm 2: movdqa y3=%xmm0 +movdqa %xmm6,%xmm0 + +# qhasm: uint32323232 y3 += z15 +# asm 1: paddd r3=int6464#13 +# asm 2: movdqa r3=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y3 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y2=int6464#1 +# asm 2: movdqa y2=%xmm0 +movdqa %xmm1,%xmm0 + +# qhasm: uint32323232 y2 += z14 +# asm 1: paddd r2=int6464#13 +# asm 2: movdqa r2=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y2 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y7=int6464#1 +# asm 2: movdqa y7=%xmm0 +movdqa %xmm2,%xmm0 + +# qhasm: uint32323232 y7 += z3 +# asm 1: paddd r7=int6464#13 +# asm 2: movdqa r7=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y7 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y6=int6464#1 +# asm 2: movdqa y6=%xmm0 +movdqa %xmm3,%xmm0 + +# qhasm: uint32323232 y6 += z2 +# asm 1: paddd r6=int6464#13 +# asm 2: movdqa r6=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y11=int6464#1 +# asm 2: movdqa y11=%xmm0 +movdqa %xmm4,%xmm0 + +# qhasm: uint32323232 y11 += z7 +# asm 1: paddd r11=int6464#13 +# asm 2: movdqa r11=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y11 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y10=int6464#1 +# asm 2: movdqa y10=%xmm0 +movdqa %xmm10,%xmm0 + +# qhasm: uint32323232 y10 += z6 +# asm 1: paddd r10=int6464#13 +# asm 2: movdqa r10=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y10 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z0=int6464#1 +# asm 2: movdqa z0=%xmm0 +movdqa 320(%rsp),%xmm0 + +# qhasm: z10_stack = z10 +# asm 1: movdqa z10_stack=stack128#21 +# asm 2: movdqa z10_stack=320(%rsp) +movdqa %xmm1,320(%rsp) + +# qhasm: y1 = z3 +# asm 1: movdqa y1=int6464#2 +# asm 2: movdqa y1=%xmm1 +movdqa %xmm4,%xmm1 + +# qhasm: uint32323232 y1 += z0 +# asm 1: paddd r1=int6464#13 +# asm 2: movdqa r1=%xmm12 +movdqa %xmm1,%xmm12 + +# qhasm: uint32323232 y1 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y15=int6464#2 +# asm 2: movdqa y15=%xmm1 +movdqa %xmm8,%xmm1 + +# qhasm: uint32323232 y15 += z11 +# asm 1: paddd r15=int6464#13 +# asm 2: movdqa r15=%xmm12 +movdqa %xmm1,%xmm12 + +# qhasm: uint32323232 y15 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z5=int6464#13 +# asm 2: movdqa z5=%xmm12 +movdqa 336(%rsp),%xmm12 + +# qhasm: z15_stack = z15 +# asm 1: movdqa z15_stack=stack128#22 +# asm 2: movdqa z15_stack=336(%rsp) +movdqa %xmm2,336(%rsp) + +# qhasm: y6 = z4 +# asm 1: movdqa y6=int6464#2 +# asm 2: movdqa y6=%xmm1 +movdqa %xmm14,%xmm1 + +# qhasm: uint32323232 y6 += z5 +# asm 1: paddd r6=int6464#3 +# asm 2: movdqa r6=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y6 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y2=int6464#2 +# asm 2: movdqa y2=%xmm1 +movdqa %xmm0,%xmm1 + +# qhasm: uint32323232 y2 += z1 +# asm 1: paddd r2=int6464#3 +# asm 2: movdqa r2=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y2 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y7=int6464#2 +# asm 2: movdqa y7=%xmm1 +movdqa %xmm12,%xmm1 + +# qhasm: uint32323232 y7 += z6 +# asm 1: paddd r7=int6464#3 +# asm 2: movdqa r7=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y7 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y3=int6464#2 +# asm 2: movdqa y3=%xmm1 +movdqa %xmm7,%xmm1 + +# qhasm: uint32323232 y3 += z2 +# asm 1: paddd r3=int6464#3 +# asm 2: movdqa r3=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y3 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y4=int6464#2 +# asm 2: movdqa y4=%xmm1 +movdqa %xmm5,%xmm1 + +# qhasm: uint32323232 y4 += z7 +# asm 1: paddd r4=int6464#3 +# asm 2: movdqa r4=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y4 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y0=int6464#2 +# asm 2: movdqa y0=%xmm1 +movdqa %xmm10,%xmm1 + +# qhasm: uint32323232 y0 += z3 +# asm 1: paddd r0=int6464#3 +# asm 2: movdqa r0=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y0 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 320(%rsp),%xmm1 + +# qhasm: z0_stack = z0 +# asm 1: movdqa z0_stack=stack128#21 +# asm 2: movdqa z0_stack=320(%rsp) +movdqa %xmm0,320(%rsp) + +# qhasm: y5 = z7 +# asm 1: movdqa y5=int6464#1 +# asm 2: movdqa y5=%xmm0 +movdqa %xmm8,%xmm0 + +# qhasm: uint32323232 y5 += z4 +# asm 1: paddd r5=int6464#3 +# asm 2: movdqa r5=%xmm2 +movdqa %xmm0,%xmm2 + +# qhasm: uint32323232 y5 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y11=int6464#1 +# asm 2: movdqa y11=%xmm0 +movdqa %xmm11,%xmm0 + +# qhasm: uint32323232 y11 += z10 +# asm 1: paddd r11=int6464#3 +# asm 2: movdqa r11=%xmm2 +movdqa %xmm0,%xmm2 + +# qhasm: uint32323232 y11 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 336(%rsp),%xmm2 + +# qhasm: z5_stack = z5 +# asm 1: movdqa z5_stack=stack128#22 +# asm 2: movdqa z5_stack=336(%rsp) +movdqa %xmm12,336(%rsp) + +# qhasm: y12 = z14 +# asm 1: movdqa y12=int6464#1 +# asm 2: movdqa y12=%xmm0 +movdqa %xmm3,%xmm0 + +# qhasm: uint32323232 y12 += z15 +# asm 1: paddd r12=int6464#13 +# asm 2: movdqa r12=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y12 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y8=int6464#1 +# asm 2: movdqa y8=%xmm0 +movdqa %xmm1,%xmm0 + +# qhasm: uint32323232 y8 += z11 +# asm 1: paddd r8=int6464#13 +# asm 2: movdqa r8=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y8 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y13=int6464#1 +# asm 2: movdqa y13=%xmm0 +movdqa %xmm2,%xmm0 + +# qhasm: uint32323232 y13 += z12 +# asm 1: paddd r13=int6464#13 +# asm 2: movdqa r13=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y13 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y9=int6464#1 +# asm 2: movdqa y9=%xmm0 +movdqa %xmm6,%xmm0 + +# qhasm: uint32323232 y9 += z8 +# asm 1: paddd r9=int6464#13 +# asm 2: movdqa r9=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y9 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y14=int6464#1 +# asm 2: movdqa y14=%xmm0 +movdqa %xmm13,%xmm0 + +# qhasm: uint32323232 y14 += z13 +# asm 1: paddd r14=int6464#13 +# asm 2: movdqa r14=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y14 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y10=int6464#1 +# asm 2: movdqa y10=%xmm0 +movdqa %xmm15,%xmm0 + +# qhasm: uint32323232 y10 += z9 +# asm 1: paddd r10=int6464#13 +# asm 2: movdqa r10=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y10 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y15=int6464#1 +# asm 2: movdqa y15=%xmm0 +movdqa %xmm9,%xmm0 + +# qhasm: uint32323232 y15 += z14 +# asm 1: paddd r15=int6464#13 +# asm 2: movdqa r15=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y15 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z0=int6464#13 +# asm 2: movdqa z0=%xmm12 +movdqa 320(%rsp),%xmm12 + +# qhasm: z5 = z5_stack +# asm 1: movdqa z5=int6464#1 +# asm 2: movdqa z5=%xmm0 +movdqa 336(%rsp),%xmm0 + +# qhasm: unsigned>? i -= 2 +# asm 1: sub $2, +ja ._mainloop1 + +# qhasm: uint32323232 z0 += orig0 +# asm 1: paddd in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: (uint32) in0 ^= *(uint32 *) (m + 192) +# asm 1: xorl 192(in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: (uint32) in4 ^= *(uint32 *) (m + 208) +# asm 1: xorl 208(in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: (uint32) in8 ^= *(uint32 *) (m + 224) +# asm 1: xorl 224(in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: (uint32) in12 ^= *(uint32 *) (m + 240) +# asm 1: xorl 240(bytes=int64#6 +# asm 2: movq bytes=%r9 +movq 408(%rsp),%r9 + +# qhasm: bytes -= 256 +# asm 1: sub $256,? bytes - 0 +# asm 1: cmp $0, +jbe ._done +# comment:fp stack unchanged by fallthrough + +# qhasm: bytesbetween1and255: +._bytesbetween1and255: + +# qhasm: unsignedctarget=int64#3 +# asm 2: mov ctarget=%rdx +mov %rdi,%rdx + +# qhasm: out = &tmp +# asm 1: leaq out=int64#1 +# asm 2: leaq out=%rdi +leaq 416(%rsp),%rdi + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = *m++; --i } +rep movsb + +# qhasm: out = &tmp +# asm 1: leaq out=int64#1 +# asm 2: leaq out=%rdi +leaq 416(%rsp),%rdi + +# qhasm: m = &tmp +# asm 1: leaq m=int64#2 +# asm 2: leaq m=%rsi +leaq 416(%rsp),%rsi +# comment:fp stack unchanged by fallthrough + +# qhasm: nocopy: +._nocopy: + +# qhasm: bytes_backup = bytes +# asm 1: movq bytes_backup=stack64#8 +# asm 2: movq bytes_backup=408(%rsp) +movq %r9,408(%rsp) + +# qhasm: diag0 = x0 +# asm 1: movdqa diag0=int6464#1 +# asm 2: movdqa diag0=%xmm0 +movdqa 48(%rsp),%xmm0 + +# qhasm: diag1 = x1 +# asm 1: movdqa diag1=int6464#2 +# asm 2: movdqa diag1=%xmm1 +movdqa 0(%rsp),%xmm1 + +# qhasm: diag2 = x2 +# asm 1: movdqa diag2=int6464#3 +# asm 2: movdqa diag2=%xmm2 +movdqa 16(%rsp),%xmm2 + +# qhasm: diag3 = x3 +# asm 1: movdqa diag3=int6464#4 +# asm 2: movdqa diag3=%xmm3 +movdqa 32(%rsp),%xmm3 + +# qhasm: a0 = diag1 +# asm 1: movdqa a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: i = 12 +# asm 1: mov $12,>i=int64#4 +# asm 2: mov $12,>i=%rcx +mov $12,%rcx + +# qhasm: mainloop2: +._mainloop2: + +# qhasm: uint32323232 a0 += diag0 +# asm 1: paddd a1=int6464#6 +# asm 2: movdqa a1=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b0 = a0 +# asm 1: movdqa b0=int6464#7 +# asm 2: movdqa b0=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a0 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a2=int6464#5 +# asm 2: movdqa a2=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b1 = a1 +# asm 1: movdqa b1=int6464#7 +# asm 2: movdqa b1=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a1 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a3=int6464#6 +# asm 2: movdqa a3=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b2 = a2 +# asm 1: movdqa b2=int6464#7 +# asm 2: movdqa b2=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a2 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a4=int6464#5 +# asm 2: movdqa a4=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b3 = a3 +# asm 1: movdqa b3=int6464#7 +# asm 2: movdqa b3=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a3 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a5=int6464#6 +# asm 2: movdqa a5=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b4 = a4 +# asm 1: movdqa b4=int6464#7 +# asm 2: movdqa b4=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a6=int6464#5 +# asm 2: movdqa a6=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b5 = a5 +# asm 1: movdqa b5=int6464#7 +# asm 2: movdqa b5=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a5 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a7=int6464#6 +# asm 2: movdqa a7=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b6 = a6 +# asm 1: movdqa b6=int6464#7 +# asm 2: movdqa b6=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b7 = a7 +# asm 1: movdqa b7=int6464#7 +# asm 2: movdqa b7=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a7 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a1=int6464#6 +# asm 2: movdqa a1=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b0 = a0 +# asm 1: movdqa b0=int6464#7 +# asm 2: movdqa b0=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a0 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a2=int6464#5 +# asm 2: movdqa a2=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b1 = a1 +# asm 1: movdqa b1=int6464#7 +# asm 2: movdqa b1=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a1 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a3=int6464#6 +# asm 2: movdqa a3=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b2 = a2 +# asm 1: movdqa b2=int6464#7 +# asm 2: movdqa b2=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a2 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a4=int6464#5 +# asm 2: movdqa a4=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b3 = a3 +# asm 1: movdqa b3=int6464#7 +# asm 2: movdqa b3=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a3 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a5=int6464#6 +# asm 2: movdqa a5=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b4 = a4 +# asm 1: movdqa b4=int6464#7 +# asm 2: movdqa b4=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a6=int6464#5 +# asm 2: movdqa a6=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b5 = a5 +# asm 1: movdqa b5=int6464#7 +# asm 2: movdqa b5=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a5 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a7=int6464#6 +# asm 2: movdqa a7=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b6 = a6 +# asm 1: movdqa b6=int6464#7 +# asm 2: movdqa b6=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,? i -= 4 +# asm 1: sub $4,a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b7 = a7 +# asm 1: movdqa b7=int6464#7 +# asm 2: movdqa b7=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a7 <<= 18 +# asm 1: pslld $18,b0=int6464#8,>b0=int6464#8 +# asm 2: pxor >b0=%xmm7,>b0=%xmm7 +pxor %xmm7,%xmm7 + +# qhasm: uint32323232 b7 >>= 14 +# asm 1: psrld $14, +ja ._mainloop2 + +# qhasm: uint32323232 diag0 += x0 +# asm 1: paddd in0=int64#4 +# asm 2: movd in0=%rcx +movd %xmm0,%rcx + +# qhasm: in12 = diag1 +# asm 1: movd in12=int64#5 +# asm 2: movd in12=%r8 +movd %xmm1,%r8 + +# qhasm: in8 = diag2 +# asm 1: movd in8=int64#6 +# asm 2: movd in8=%r9 +movd %xmm2,%r9 + +# qhasm: in4 = diag3 +# asm 1: movd in4=int64#7 +# asm 2: movd in4=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in1 = diag1 +# asm 1: movd in1=int64#5 +# asm 2: movd in1=%r8 +movd %xmm1,%r8 + +# qhasm: in13 = diag2 +# asm 1: movd in13=int64#6 +# asm 2: movd in13=%r9 +movd %xmm2,%r9 + +# qhasm: in9 = diag3 +# asm 1: movd in9=int64#7 +# asm 2: movd in9=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in10=int64#4 +# asm 2: movd in10=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = diag1 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm1,%r8 + +# qhasm: in2 = diag2 +# asm 1: movd in2=int64#6 +# asm 2: movd in2=%r9 +movd %xmm2,%r9 + +# qhasm: in14 = diag3 +# asm 1: movd in14=int64#7 +# asm 2: movd in14=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in15=int64#4 +# asm 2: movd in15=%rcx +movd %xmm0,%rcx + +# qhasm: in11 = diag1 +# asm 1: movd in11=int64#5 +# asm 2: movd in11=%r8 +movd %xmm1,%r8 + +# qhasm: in7 = diag2 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm2,%r9 + +# qhasm: in3 = diag3 +# asm 1: movd in3=int64#7 +# asm 2: movd in3=%rax +movd %xmm3,%rax + +# qhasm: (uint32) in15 ^= *(uint32 *) (m + 60) +# asm 1: xorl 60(bytes=int64#6 +# asm 2: movq bytes=%r9 +movq 408(%rsp),%r9 + +# qhasm: in8 = ((uint32 *)&x2)[0] +# asm 1: movl in8=int64#4d +# asm 2: movl in8=%ecx +movl 16(%rsp),%ecx + +# qhasm: in9 = ((uint32 *)&x3)[1] +# asm 1: movl 4+in9=int64#5d +# asm 2: movl 4+in9=%r8d +movl 4+32(%rsp),%r8d + +# qhasm: in8 += 1 +# asm 1: add $1,in9=int64#5 +# asm 2: mov in9=%r8 +mov %rcx,%r8 + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %ecx,16(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl ? unsigned +ja ._bytesatleast65 +# comment:fp stack unchanged by jump + +# qhasm: goto bytesatleast64 if !unsigned< +jae ._bytesatleast64 + +# qhasm: m = out +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rdi,%rsi + +# qhasm: out = ctarget +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdx,%rdi + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = *m++; --i } +rep movsb +# comment:fp stack unchanged by fallthrough + +# qhasm: bytesatleast64: +._bytesatleast64: +# comment:fp stack unchanged by fallthrough + +# qhasm: done: +._done: + +# qhasm: r11_caller = r11_stack +# asm 1: movq r11_caller=int64#9 +# asm 2: movq r11_caller=%r11 +movq 352(%rsp),%r11 + +# qhasm: r12_caller = r12_stack +# asm 1: movq r12_caller=int64#10 +# asm 2: movq r12_caller=%r12 +movq 360(%rsp),%r12 + +# qhasm: r13_caller = r13_stack +# asm 1: movq r13_caller=int64#11 +# asm 2: movq r13_caller=%r13 +movq 368(%rsp),%r13 + +# qhasm: r14_caller = r14_stack +# asm 1: movq r14_caller=int64#12 +# asm 2: movq r14_caller=%r14 +movq 376(%rsp),%r14 + +# qhasm: r15_caller = r15_stack +# asm 1: movq r15_caller=int64#13 +# asm 2: movq r15_caller=%r15 +movq 384(%rsp),%r15 + +# qhasm: rbx_caller = rbx_stack +# asm 1: movq rbx_caller=int64#14 +# asm 2: movq rbx_caller=%rbx +movq 392(%rsp),%rbx + +# qhasm: rbp_caller = rbp_stack +# asm 1: movq rbp_caller=int64#15 +# asm 2: movq rbp_caller=%rbp +movq 400(%rsp),%rbp + +# qhasm: leave +add %r11,%rsp +xor %rax,%rax +xor %rdx,%rdx +ret + +# qhasm: bytesatleast65: +._bytesatleast65: + +# qhasm: bytes -= 64 +# asm 1: sub $64, @@ -1064,7 +1068,7 @@ const char *Packet::errorString(ErrorCode e) void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) { - uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t mangledKey[32]; uint8_t *const data = reinterpret_cast(unsafeData()); // Mask least significant 3 bits of packet ID with counter to embed packet send counter for QoS use @@ -1074,23 +1078,47 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE); _salsa20MangleKey((const unsigned char *)key,mangledKey); + +#ifdef ZT_USE_X64_ASM_SALSA2012 + const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),payloadLen + 64,reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); + + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } + + uint64_t mac[2]; + Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); +#else Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - // MAC key is always the first 32 bytes of the Salsa20 key stream - // This is the same construction DJB's NaCl library uses + uint64_t macKey[4]; s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); uint8_t *const payload = data + ZT_PACKET_IDX_VERB; const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; if (encryptPayload) s20.crypt12(payload,payload,payloadLen); + + uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); memcpy(data + ZT_PACKET_IDX_MAC,mac,8); +#endif } bool Packet::dearmor(const void *key) { - uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t mangledKey[32]; uint8_t *const data = reinterpret_cast(unsafeData()); const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; unsigned char *const payload = data + ZT_PACKET_IDX_VERB; @@ -1098,9 +1126,37 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); + +#ifdef ZT_USE_X64_ASM_SALSA2012 + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); + + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,keyStream); + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender + + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } + } + + return true; +#else Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + uint64_t macKey[4]; s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) return false; // MAC failed, packet is corrupt, modified, or is not from the sender @@ -1109,6 +1165,7 @@ bool Packet::dearmor(const void *key) s20.crypt12(payload,payload,payloadLen); return true; +#endif } else { return false; // unrecognized cipher suite } diff --git a/selftest.cpp b/selftest.cpp index fe0aa933..b7a1cc4d 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -54,6 +54,10 @@ #include "controller/JSONDB.hpp" +#ifdef ZT_USE_X64_ASM_SALSA2012 +#include "ext/x64-salsa2012-asm/salsa2012.h" +#endif + #ifdef __WINDOWS__ #include #endif @@ -204,6 +208,24 @@ static int testCrypto() ::free((void *)bb); } +#ifdef ZT_USE_X64_ASM_SALSA2012 + std::cout << "[crypto] Benchmarking Salsa20/12 fast x64 ASM... "; std::cout.flush(); + { + unsigned char *bb = (unsigned char *)::malloc(1234567); + for(unsigned int i=0;i<1234567;++i) + bb[i] = (unsigned char)i; + double bytes = 0.0; + uint64_t start = OSUtils::now(); + for(unsigned int i=0;i<200;++i) { + zt_salsa2012_amd64_xmm6_xor(bb,bb,1234567,s20TV0Iv,s20TV0Key); + bytes += 1234567.0; + } + uint64_t end = OSUtils::now(); + std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl; + ::free((void *)bb); + } +#endif + std::cout << "[crypto] Benchmarking Salsa20/20... "; std::cout.flush(); { unsigned char *bb = (unsigned char *)::malloc(1234567); -- cgit v1.2.3 From 72bd3064a20aec9b1fcbc86bb590aca87f4eca71 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Apr 2017 11:00:29 -0700 Subject: Windows build fixes, self test cleanup. --- node/Dictionary.hpp | 40 ---------- selftest.cpp | 99 ++++++------------------- windows/ZeroTierOne/ZeroTierOne.vcxproj | 8 +- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 21 +++++- 4 files changed, 43 insertions(+), 125 deletions(-) (limited to 'node') diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index fa9e2883..440dae7f 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -416,46 +416,6 @@ public: return (this->get(key,tmp,2) >= 0); } - /** - * Erase a key from this dictionary - * - * Use this before add() to ensure that a key is replaced if it might - * already be present. - * - * @param key Key to erase - * @return True if key was found and erased - */ - inline bool erase(const char *key) - { - char d2[C]; - char *saveptr = (char *)0; - unsigned int d2ptr = 0; - bool found = false; - for(char *f=Utils::stok(_d,"\r\n",&saveptr);(f);f=Utils::stok((char *)0,"\r\n",&saveptr)) { - if (*f) { - const char *p = f; - const char *k = key; - while ((*k)&&(*p)) { - if (*k != *p) - break; - ++k; - ++p; - } - if (*k) { - p = f; - while (*p) - d2[d2ptr++] = *(p++); - d2[d2ptr++] = '\n'; - } else { - found = true; - } - } - } - d2[d2ptr++] = (char)0; - memcpy(_d,d2,d2ptr); - return found; - } - /** * @return Value of C template parameter */ diff --git a/selftest.cpp b/selftest.cpp index b7a1cc4d..55a469e1 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -196,7 +196,7 @@ static int testCrypto() for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; Salsa20 s20(s20TV0Key,s20TV0Iv); - double bytes = 0.0; + long double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { s20.crypt12(bb,bb,1234567); @@ -204,7 +204,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; ::free((void *)bb); } @@ -221,7 +221,7 @@ static int testCrypto() bytes += 1234567.0; } uint64_t end = OSUtils::now(); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl; + std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1024.0)) << " MiB/second" << std::endl; ::free((void *)bb); } #endif @@ -232,7 +232,7 @@ static int testCrypto() for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; Salsa20 s20(s20TV0Key,s20TV0Iv); - double bytes = 0.0; + long double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { s20.crypt20(bb,bb,1234567); @@ -240,7 +240,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; ::free((void *)bb); } @@ -270,14 +270,14 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - double bytes = 0.0; + long double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { Poly1305::compute(buf1,bb,1234567,poly1305TV0Key); bytes += 1234567.0; } uint64_t end = OSUtils::now(); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1000.0)) << " MiB/second" << std::endl; ::free((void *)bb); } @@ -632,6 +632,7 @@ static int testOther() return -1; } +#if 0 std::cout << "[other] Testing Hashtable... "; std::cout.flush(); { Hashtable ht; @@ -795,40 +796,28 @@ static int testOther() } } std::cout << "PASS" << std::endl; - - std::cout << "[other] Testing hex encode/decode... "; std::cout.flush(); - for(unsigned int k=0;k<1000;++k) { - unsigned int flen = (rand() % 8194) + 1; - for(unsigned int i=0;i test; + Dictionary<8194> *test = new Dictionary<8194>(); char key[32][16]; char value[32][128]; + memset(key, 0, sizeof(key)); + memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { Utils::snprintf(key[q],16,"%.8lx",(unsigned long)rand()); - int r = rand() % 128; + int r = (rand() % 127) + 1; for(int x=0;xadd(key[q],value[q],r); } for(unsigned int q=0;q<1024;++q) { //int r = rand() % 128; int r = 31; char tmp[128]; - if (test.get(key[r],tmp,sizeof(tmp)) >= 0) { + if (test->get(key[r],tmp,sizeof(tmp)) >= 0) { if (strcmp(value[r],tmp)) { std::cout << "FAILED (invalid value)!" << std::endl; return -1; @@ -838,76 +827,30 @@ static int testOther() return -1; } } - for(unsigned int q=0;q<31;++q) { - char tmp[128]; - test.erase(key[q]); - if (test.get(key[q],tmp,sizeof(tmp)) >= 0) { - std::cout << "FAILED (key should have been erased)!" << std::endl; - return -1; - } - if (test.get(key[q+1],tmp,sizeof(tmp)) < 0) { - std::cout << "FAILED (key should NOT have been erased)!" << std::endl; - return -1; - } - } + delete test; } int foo = 0; volatile int *volatile bar = &foo; // force compiler not to optimize out test.get() below for(int k=0;k<200;++k) { int r = rand() % 8194; - unsigned char tmp[8194]; + unsigned char *tmp = new unsigned char[8194]; for(int q=0;q test((const char *)tmp); + Dictionary<8194> *test = new Dictionary<8194>((const char *)tmp); for(unsigned int q=0;q<100;++q) { char tmp[128]; for(unsigned int x=0;x<128;++x) tmp[x] = (char)(rand() & 0xff); tmp[127] = (char)0; char value[8194]; - *bar += test.get(tmp,value,sizeof(value)); + *bar += test->get(tmp,value,sizeof(value)); } + delete test; + delete[] tmp; } std::cout << "PASS (junk value to prevent optimization-out of test: " << foo << ")" << std::endl; - /* - std::cout << "[other] Testing controller/JSONDB..."; std::cout.flush(); - { - std::map db1data; - JSONDB db1("jsondb-test"); - for(unsigned int i=0;i<256;++i) { - std::string n; - for(unsigned int j=0,k=rand() % 4;j<=k;++j) { - if (j > 0) n.push_back('/'); - char foo[24]; - Utils::snprintf(foo,sizeof(foo),"%lx",rand()); - n.append(foo); - } - db1data[n] = {{"i",i}}; - db1.put(n,db1data[n]); - } - for(std::map::iterator i(db1data.begin());i!=db1data.end();++i) { - i->second["foo"] = "bar"; - db1.put(i->first,i->second); - } - JSONDB db2("jsondb-test"); - if (db1 != db2) { - std::cout << " FAILED (db1!=db2 #1)" << std::endl; - return -1; - } - for(std::map::iterator i(db1data.begin());i!=db1data.end();++i) { - db1.erase(i->first); - } - db2.reload(); - if (db1 != db2) { - std::cout << " FAILED (db1!=db2 #2)" << std::endl; - return -1; - } - } - std::cout << " PASS" << std::endl; - */ - return 0; } diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 84a44198..6ed95010 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -72,10 +72,7 @@ - - false - false - + @@ -88,6 +85,8 @@ + + @@ -109,6 +108,7 @@ + diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index b051b4f7..01637801 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -76,6 +76,12 @@ {ff20532b-d9a2-440d-a7b4-b49e26a9b2f8} + + {05d9cde8-03ae-4e37-b9f7-7417de98cbe9} + + + {7dc22e9c-f869-41e7-b43d-f07f5b94f6fb} + @@ -150,9 +156,6 @@ Source Files\ext\http-parser - - Source Files - Source Files\windows\ZeroTierOne @@ -246,6 +249,9 @@ Source Files\node + + Source Files + @@ -485,6 +491,15 @@ Header Files\node + + Header Files\ext\x64-salsa2012-asm + + + Header Files\controller + + + Header Files\controller + -- cgit v1.2.3 From aaf597f0205b79c0f1d8523f15c8d8aaecb63f43 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Apr 2017 12:22:44 -0700 Subject: Cleanup, Windows fixes, Self test fix --- ext/x64-salsa2012-asm/salsa2012.h | 6 ++---- node/Dictionary.hpp | 18 +++++++++--------- one.cpp | 2 +- selftest.cpp | 15 ++++++--------- windows/ZeroTierOne/ZeroTierOne.vcxproj | 22 ++++++++++++++++++++-- windows/ZeroTierOne/ZeroTierOne.vcxproj.filters | 3 +++ 6 files changed, 41 insertions(+), 25 deletions(-) (limited to 'node') diff --git a/ext/x64-salsa2012-asm/salsa2012.h b/ext/x64-salsa2012-asm/salsa2012.h index d47059b4..d8c2e48c 100644 --- a/ext/x64-salsa2012-asm/salsa2012.h +++ b/ext/x64-salsa2012-asm/salsa2012.h @@ -2,11 +2,9 @@ extern "C" { #endif +// Generates Salsa20/12 key stream // output, outlen, nonce, key (256-bit / 32-byte) -extern int zt_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); - -// ciphertext, message, mlen, nonce, key -extern int zt_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int zt_salsa2012_amd64_xmm6(unsigned char *, unsigned long long, const unsigned char *, const unsigned char *); #ifdef __cplusplus } diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 440dae7f..0db13b63 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -176,12 +176,12 @@ public: j = 0; esc = false; ++p; - while ((*p != 0)&&(*p != '\r')&&(*p != '\n')) { + while ((*p != 0)&&(*p != 13)&&(*p != 10)) { if (esc) { esc = false; switch(*p) { - case 'r': dest[j++] = '\r'; break; - case 'n': dest[j++] = '\n'; break; + case 'r': dest[j++] = 13; break; + case 'n': dest[j++] = 10; break; case '0': dest[j++] = (char)0; break; case 'e': dest[j++] = '='; break; default: dest[j++] = *p; break; @@ -207,7 +207,7 @@ public: dest[j] = (char)0; return j; } else { - while ((*p)&&(*p != '\r')&&(*p != '\n')) { + while ((*p)&&(*p != 13)&&(*p != 10)) { if (++p == eof) { dest[0] = (char)0; return -1; @@ -299,7 +299,7 @@ public: unsigned int j = i; if (j > 0) { - _d[j++] = '\n'; + _d[j++] = (char)10; if (j == C) { _d[i] = (char)0; return false; @@ -326,8 +326,8 @@ public: while ( ((vlen < 0)&&(*p)) || (k < vlen) ) { switch(*p) { case 0: - case '\r': - case '\n': + case 13: + case 10: case '\\': case '=': _d[j++] = '\\'; @@ -337,8 +337,8 @@ public: } switch(*p) { case 0: _d[j++] = '0'; break; - case '\r': _d[j++] = 'r'; break; - case '\n': _d[j++] = 'n'; break; + case 13: _d[j++] = 'r'; break; + case 10: _d[j++] = 'n'; break; case '\\': _d[j++] = '\\'; break; case '=': _d[j++] = 'e'; break; } diff --git a/one.cpp b/one.cpp index edefe82e..b40e28fc 100644 --- a/one.cpp +++ b/one.cpp @@ -1257,7 +1257,7 @@ public: }; #ifdef __WINDOWS__ -int _tmain(int argc, _TCHAR* argv[]) +int __cdecl _tmain(int argc, _TCHAR* argv[]) #else int main(int argc,char **argv) #endif diff --git a/selftest.cpp b/selftest.cpp index 55a469e1..e68cf047 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -212,12 +212,10 @@ static int testCrypto() std::cout << "[crypto] Benchmarking Salsa20/12 fast x64 ASM... "; std::cout.flush(); { unsigned char *bb = (unsigned char *)::malloc(1234567); - for(unsigned int i=0;i<1234567;++i) - bb[i] = (unsigned char)i; double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { - zt_salsa2012_amd64_xmm6_xor(bb,bb,1234567,s20TV0Iv,s20TV0Key); + zt_salsa2012_amd64_xmm6(bb, 1234567, s20TV0Iv, s20TV0Key); bytes += 1234567.0; } uint64_t end = OSUtils::now(); @@ -806,20 +804,19 @@ static int testOther() memset(key, 0, sizeof(key)); memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - Utils::snprintf(key[q],16,"%.8lx",(unsigned long)rand()); - int r = (rand() % 127) + 1; + Utils::snprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); + int r = rand() % 128; for(int x=0;xadd(key[q],value[q],r); } for(unsigned int q=0;q<1024;++q) { - //int r = rand() % 128; - int r = 31; + int r = rand() % 32; char tmp[128]; if (test->get(key[r],tmp,sizeof(tmp)) >= 0) { if (strcmp(value[r],tmp)) { - std::cout << "FAILED (invalid value)!" << std::endl; + std::cout << "FAILED (invalid value '" << value[r] << "' != '" << tmp << "')!" << std::endl; return -1; } } else { @@ -1048,7 +1045,7 @@ static int testHttp() */ #ifdef __WINDOWS__ -int _tmain(int argc, _TCHAR* argv[]) +int __cdecl _tmain(int argc, _TCHAR* argv[]) #else int main(int argc,char **argv) #endif diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 6ed95010..96de5d2b 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -72,12 +72,27 @@ - + + false + false + false + false + false + false + + + true + true + true + true + true + true + @@ -363,7 +378,7 @@ Level3 - Full + MaxSpeed true true true @@ -377,6 +392,9 @@ Speed true 4996 + Guard + false + VectorCall true diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index 01637801..cca10f97 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -252,6 +252,9 @@ Source Files + + Source Files + -- cgit v1.2.3 From e7a2c6ecefc95422145385fdfd3ff137e5d290ac Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 19 Apr 2017 17:11:56 -0700 Subject: Integrate ARM/NEON crypto. --- ext/arm32-neon-salsa2012-asm/salsa2012.h | 2 +- node/Packet.cpp | 161 ++++++++++++++++++------------- 2 files changed, 96 insertions(+), 67 deletions(-) (limited to 'node') diff --git a/ext/arm32-neon-salsa2012-asm/salsa2012.h b/ext/arm32-neon-salsa2012-asm/salsa2012.h index 719b2e0c..95b247f2 100644 --- a/ext/arm32-neon-salsa2012-asm/salsa2012.h +++ b/ext/arm32-neon-salsa2012-asm/salsa2012.h @@ -4,7 +4,7 @@ #if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux) #include #include -#define zt_arm_has_neon() (getauxval(AT_HWCAP) & HWCAP_NEON) +#define zt_arm_has_neon() ((getauxval(AT_HWCAP) & HWCAP_NEON) != 0) #else #define zt_arm_has_neon() (true) #endif diff --git a/node/Packet.cpp b/node/Packet.cpp index 39a0a4d5..85b18cff 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -27,6 +27,9 @@ #ifdef ZT_USE_X64_ASM_SALSA2012 #include "../ext/x64-salsa2012-asm/salsa2012.h" #endif +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +#include "../ext/arm32-neon-salsa2012-asm/salsa2012.h" +#endif #ifdef _MSC_VER #define FORCE_INLINE static __forceinline @@ -40,6 +43,34 @@ namespace ZeroTier { /************************************************************************** */ + +/* Set up macros for fast single-pass ASM Salsa20/12 crypto, if we have it */ + +// x64 SSE crypto +#ifdef ZT_USE_X64_ASM_SALSA2012 +#define ZT_HAS_FAST_CRYPTO() (true) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) zt_salsa2012_amd64_xmm6(reinterpret_cast(b),(l),reinterpret_cast(n),reinterpret_cast(k)) +#endif + +// ARM (32-bit) NEON crypto (must be detected) +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +class _FastCryptoChecker +{ +public: + _FastCryptoChecker() : canHas(zt_arm_has_neon()) {} + bool canHas; +}; +static const _FastCryptoChecker _ZT_FAST_CRYPTO_CHECK; +#define ZT_HAS_FAST_CRYPTO() (_ZT_FAST_CRYPTO_CHECK.canHas) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) zt_salsa2012_armneon3_xor(reinterpret_cast(b),(const unsigned char *)0,(l),reinterpret_cast(n),reinterpret_cast(k)) +#endif + +// No fast crypto available +#ifndef ZT_HAS_FAST_CRYPTO +#define ZT_HAS_FAST_CRYPTO() (false) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) {} +#endif + /************************************************************************** */ /* LZ4 is shipped encapsulated into Packet in an anonymous namespace. @@ -1079,41 +1110,41 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) _salsa20MangleKey((const unsigned char *)key,mangledKey); -#ifdef ZT_USE_X64_ASM_SALSA2012 - const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; - uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; - zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),payloadLen + 64,reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); - - uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block - uint8_t *dptr = data + ZT_PACKET_IDX_VERB; - unsigned int ksrem = payloadLen; - while (ksrem >= 8) { - ksrem -= 8; - *(reinterpret_cast(dptr)) ^= *(ksptr++); - dptr += 8; - } - for(unsigned int i=0;i(ksptr)[i]; - } + if (ZT_HAS_FAST_CRYPTO()) { + const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,payloadLen + 64,(data + ZT_PACKET_IDX_IV),mangledKey); + + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } - uint64_t mac[2]; - Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); - memcpy(data + ZT_PACKET_IDX_MAC,mac,8); -#else - Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + uint64_t mac[2]; + Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); + } else { + Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - uint64_t macKey[4]; - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t macKey[4]; + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - uint8_t *const payload = data + ZT_PACKET_IDX_VERB; - const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; - if (encryptPayload) - s20.crypt12(payload,payload,payloadLen); + uint8_t *const payload = data + ZT_PACKET_IDX_VERB; + const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; + if (encryptPayload) + s20.crypt12(payload,payload,payloadLen); - uint64_t mac[2]; - Poly1305::compute(mac,payload,payloadLen,macKey); - memcpy(data + ZT_PACKET_IDX_MAC,mac,8); -#endif + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,macKey); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); + } } bool Packet::dearmor(const void *key) @@ -1127,45 +1158,43 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); -#ifdef ZT_USE_X64_ASM_SALSA2012 - uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; - zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); - - uint64_t mac[2]; - Poly1305::compute(mac,payload,payloadLen,keyStream); - if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - - if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { - uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block - uint8_t *dptr = data + ZT_PACKET_IDX_VERB; - unsigned int ksrem = payloadLen; - while (ksrem >= 8) { - ksrem -= 8; - *(reinterpret_cast(dptr)) ^= *(ksptr++); - dptr += 8; - } - for(unsigned int i=0;i(ksptr)[i]; + if (ZT_HAS_FAST_CRYPTO()) { + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),(data + ZT_PACKET_IDX_IV),mangledKey); + + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,keyStream); + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender + + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } } + } else { + Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + + uint64_t macKey[4]; + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,macKey); + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender + + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) + s20.crypt12(payload,payload,payloadLen); } return true; -#else - Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - - uint64_t macKey[4]; - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - uint64_t mac[2]; - Poly1305::compute(mac,payload,payloadLen,macKey); - if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - - if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) - s20.crypt12(payload,payload,payloadLen); - - return true; -#endif } else { return false; // unrecognized cipher suite } -- cgit v1.2.3 From a8ced184dc67c5cf39ce1332156c7eb80241768b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 20 Apr 2017 09:33:35 -0700 Subject: Some code cleanup and make sure any type punning is guarded with ZT_NO_TYPE_PUNNING. --- node/Packet.cpp | 60 ++++++++++++++++++++------------------------------------ node/Salsa20.cpp | 40 +++++++++++++++++-------------------- node/Salsa20.hpp | 37 ++++++++++++++++++++++++++++++++++ selftest.cpp | 2 -- 4 files changed, 76 insertions(+), 63 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index 85b18cff..8a57dd55 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1109,38 +1109,26 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE); _salsa20MangleKey((const unsigned char *)key,mangledKey); - if (ZT_HAS_FAST_CRYPTO()) { - const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; + const unsigned int encryptLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; - ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,payloadLen + 64,(data + ZT_PACKET_IDX_IV),mangledKey); - - uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block - uint8_t *dptr = data + ZT_PACKET_IDX_VERB; - unsigned int ksrem = payloadLen; - while (ksrem >= 8) { - ksrem -= 8; - *(reinterpret_cast(dptr)) ^= *(ksptr++); - dptr += 8; - } - for(unsigned int i=0;i(ksptr)[i]; - } - + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,encryptLen + 64,(data + ZT_PACKET_IDX_IV),mangledKey); + Salsa20::memxor(data + ZT_PACKET_IDX_VERB,reinterpret_cast(keyStream + 8),encryptLen); uint64_t mac[2]; Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); +#ifdef ZT_NO_TYPE_PUNNING memcpy(data + ZT_PACKET_IDX_MAC,mac,8); +#else + (*reinterpret_cast(data + ZT_PACKET_IDX_MAC)) = mac[0]; +#endif } else { Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - uint64_t macKey[4]; s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - uint8_t *const payload = data + ZT_PACKET_IDX_VERB; const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; if (encryptPayload) s20.crypt12(payload,payload,payloadLen); - uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); memcpy(data + ZT_PACKET_IDX_MAC,mac,8); @@ -1157,39 +1145,33 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); - if (ZT_HAS_FAST_CRYPTO()) { uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),(data + ZT_PACKET_IDX_IV),mangledKey); - uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,keyStream); +#ifdef ZT_NO_TYPE_PUNNING if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - - if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { - uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block - uint8_t *dptr = data + ZT_PACKET_IDX_VERB; - unsigned int ksrem = payloadLen; - while (ksrem >= 8) { - ksrem -= 8; - *(reinterpret_cast(dptr)) ^= *(ksptr++); - dptr += 8; - } - for(unsigned int i=0;i(ksptr)[i]; - } - } + return false; +#else + if ((*reinterpret_cast(data + ZT_PACKET_IDX_MAC)) != mac[0]) // also secure, constant time + return false; +#endif + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) + Salsa20::memxor(data + ZT_PACKET_IDX_VERB,reinterpret_cast(keyStream + 8),payloadLen); } else { Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - uint64_t macKey[4]; s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); +#ifdef ZT_NO_TYPE_PUNNING if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - + return false; +#else + if ((*reinterpret_cast(data + ZT_PACKET_IDX_MAC)) != mac[0]) // also secure, constant time + return false; +#endif if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) s20.crypt12(payload,payload,payloadLen); } diff --git a/node/Salsa20.cpp b/node/Salsa20.cpp index 2a802555..1d4117e3 100644 --- a/node/Salsa20.cpp +++ b/node/Salsa20.cpp @@ -69,46 +69,42 @@ namespace ZeroTier { void Salsa20::init(const void *key,const void *iv) { #ifdef ZT_SALSA20_SSE - const uint32_t *k = (const uint32_t *)key; - + const uint32_t *const k = (const uint32_t *)key; _state.i[0] = 0x61707865; - _state.i[3] = 0x6b206574; - _state.i[13] = k[0]; - _state.i[10] = k[1]; - _state.i[7] = k[2]; - _state.i[4] = k[3]; - k += 4; _state.i[1] = 0x3320646e; _state.i[2] = 0x79622d32; - _state.i[15] = k[0]; - _state.i[12] = k[1]; - _state.i[9] = k[2]; - _state.i[6] = k[3]; - _state.i[14] = ((const uint32_t *)iv)[0]; - _state.i[11] = ((const uint32_t *)iv)[1]; + _state.i[3] = 0x6b206574; + _state.i[4] = k[3]; _state.i[5] = 0; + _state.i[6] = k[7]; + _state.i[7] = k[2]; _state.i[8] = 0; + _state.i[9] = k[6]; + _state.i[10] = k[1]; + _state.i[11] = ((const uint32_t *)iv)[1]; + _state.i[12] = k[5]; + _state.i[13] = k[0]; + _state.i[14] = ((const uint32_t *)iv)[0]; + _state.i[15] = k[4]; #else const char *const constants = "expand 32-byte k"; - const uint8_t *k = (const uint8_t *)key; - + const uint8_t *const k = (const uint8_t *)key; + _state.i[0] = U8TO32_LITTLE(constants + 0); _state.i[1] = U8TO32_LITTLE(k + 0); _state.i[2] = U8TO32_LITTLE(k + 4); _state.i[3] = U8TO32_LITTLE(k + 8); _state.i[4] = U8TO32_LITTLE(k + 12); - k += 16; _state.i[5] = U8TO32_LITTLE(constants + 4); _state.i[6] = U8TO32_LITTLE(((const uint8_t *)iv) + 0); _state.i[7] = U8TO32_LITTLE(((const uint8_t *)iv) + 4); _state.i[8] = 0; _state.i[9] = 0; _state.i[10] = U8TO32_LITTLE(constants + 8); - _state.i[11] = U8TO32_LITTLE(k + 0); - _state.i[12] = U8TO32_LITTLE(k + 4); - _state.i[13] = U8TO32_LITTLE(k + 8); - _state.i[14] = U8TO32_LITTLE(k + 12); + _state.i[11] = U8TO32_LITTLE(k + 16); + _state.i[12] = U8TO32_LITTLE(k + 20); + _state.i[13] = U8TO32_LITTLE(k + 24); + _state.i[14] = U8TO32_LITTLE(k + 28); _state.i[15] = U8TO32_LITTLE(constants + 12); - _state.i[0] = U8TO32_LITTLE(constants + 0); #endif } diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 448c8f1a..52592602 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -34,6 +34,43 @@ public: Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } + /** + * XOR d with s + * + * This is done efficiently using e.g. SSE if available. It's used when + * alternative Salsa20 implementations are used in Packet and is here + * since this is where all the SSE stuff is already included. + * + * @param d Destination to XOR + * @param s Source bytes to XOR with destination + * @param len Length of s and d + */ + static inline void memxor(uint8_t *d,const uint8_t *s,unsigned int len) + { +#ifdef ZT_SALSA20_SSE + while (len >= 16) { + _mm_storeu_si128(reinterpret_cast<__m128i *>(d),_mm_xor_si128(_mm_loadu_si128(reinterpret_cast<__m128i *>(d)),_mm_loadu_si128(reinterpret_cast(s)))); + s += 16; + d += 16; + len -= 16; + } +#else +#ifndef ZT_NO_TYPE_PUNNING + while (len >= 16) { + (*reinterpret_cast(d)) ^= (*reinterpret_cast(s)); + s += 8; + d += 8; + (*reinterpret_cast(d)) ^= (*reinterpret_cast(s)); + s += 8; + d += 8; + len -= 16; + } +#endif +#endif + while (len--) + *(d++) ^= *(s++); + } + /** * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector diff --git a/selftest.cpp b/selftest.cpp index 6df2e272..91d304a6 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -141,8 +141,6 @@ static const C25519TestVector C25519_TEST_VECTORS[ZT_NUM_C25519_TEST_VECTORS] = ////////////////////////////////////////////////////////////////////////////// -static unsigned char fuzzbuf[1048576]; - static int testCrypto() { unsigned char buf1[16384]; -- cgit v1.2.3 From 4f2a77976965f19640416afca24367d3037820b8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 24 Apr 2017 20:51:02 -0700 Subject: JSONDB performance improvements, threading fix. --- controller/EmbeddedNetworkController.cpp | 59 +++++++++++++++++--------------- controller/EmbeddedNetworkController.hpp | 1 + controller/JSONDB.cpp | 57 ++++++++++++++++-------------- controller/JSONDB.hpp | 31 ++++++++--------- node/Node.cpp | 3 +- osdep/Thread.hpp | 42 +++++++++-------------- 6 files changed, 96 insertions(+), 97 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3b901afe..84906849 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -431,6 +431,7 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _startTime(OSUtils::now()), + _running(true), _db(dbPath), _node(node) { @@ -438,12 +439,19 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa EmbeddedNetworkController::~EmbeddedNetworkController() { - Mutex::Lock _l(_threads_m); - if (_threads.size() > 0) { - for(unsigned long i=0;i<(((unsigned long)_threads.size())*2);++i) + _running = false; + std::vector t; + { + Mutex::Lock _l(_threads_m); + t = _threads; + } + if (t.size() > 0) { + for(unsigned long i=0,j=(unsigned long)(t.size() * 4);i::iterator i(_threads.begin());i!=_threads.end();++i) + /* + for(std::vector::iterator i(t.begin());i!=t.end();++i) Thread::join(*i); + */ } } @@ -1111,23 +1119,23 @@ void EmbeddedNetworkController::threadMain() throw() { uint64_t lastCircuitTestCheck = 0; - for(;;) { - _RQEntry *const qe = _queue.get(); // waits on next request - if (!qe) break; // enqueue a NULL to terminate threads + _RQEntry *qe = (_RQEntry *)0; + while ((_running)&&((qe = _queue.get()))) { try { _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); } catch ( ... ) {} delete qe; - - uint64_t now = OSUtils::now(); - if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - lastCircuitTestCheck = now; - Mutex::Lock _l(_tests_m); - for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { - if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - _node->circuitTestEnd(&(*i)); - _tests.erase(i++); - } else ++i; + if (_running) { + uint64_t now = OSUtils::now(); + if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + lastCircuitTestCheck = now; + Mutex::Lock _l(_tests_m); + for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { + if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + _node->circuitTestEnd(&(*i)); + _tests.erase(i++); + } else ++i; + } } } } @@ -1723,13 +1731,11 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid char pfx[256]; Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid); - { - Mutex::Lock _l(_nmiCache_m); - std::map::iterator c(_nmiCache.find(nwid)); - if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks - nmi = c->second; - return; - } + Mutex::Lock _l(_nmiCache_m); + std::map::iterator c(_nmiCache.find(nwid)); + if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks + nmi = c->second; + return; } _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { @@ -1770,10 +1776,7 @@ void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid }); nmi.nmiTimestamp = now; - { - Mutex::Lock _l(_nmiCache_m); - _nmiCache[nwid] = nmi; - } + _nmiCache[nwid] = nmi; } void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 906f4345..04f52c7d 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -178,6 +178,7 @@ private: const uint64_t _startTime; + volatile bool _running; BlockingQueue<_RQEntry *> _queue; std::vector _threads; Mutex _threads_m; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index dd8e3968..509250b9 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -50,7 +50,7 @@ JSONDB::JSONDB(const std::string &basePath) : OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } - _ready = _reload(_basePath,std::string()); + _reload(_basePath,std::string()); } bool JSONDB::writeRaw(const std::string &n,const std::string &obj) @@ -87,16 +87,16 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj) nlohmann::json JSONDB::get(const std::string &n) { - { - Mutex::Lock _l(_db_m); + while (!_ready) { + Thread::sleep(250); + _reload(_basePath,std::string()); + } - while (!_ready) { - Thread::sleep(250); - _ready = _reload(_basePath,std::string()); - } + if (!_isValidObjectName(n)) + return _EMPTY_JSON; - if (!_isValidObjectName(n)) - return _EMPTY_JSON; + { + Mutex::Lock _l(_db_m); std::map::iterator e(_db.find(n)); if (e != _db.end()) return e->second.obj; @@ -116,14 +116,16 @@ nlohmann::json JSONDB::get(const std::string &n) return _EMPTY_JSON; } - try { + { Mutex::Lock _l(_db_m); - _E &e2 = _db[n]; - e2.obj = OSUtils::jsonParse(buf); - return e2.obj; - } catch ( ... ) { - _db.erase(n); - return _EMPTY_JSON; + try { + _E &e2 = _db[n]; + e2.obj = OSUtils::jsonParse(buf); + return e2.obj; + } catch ( ... ) { + _db.erase(n); + return _EMPTY_JSON; + } } } @@ -131,7 +133,15 @@ void JSONDB::erase(const std::string &n) { if (!_isValidObjectName(n)) return; + _erase(n); + { + Mutex::Lock _l(_db_m); + _db.erase(n); + } +} +void JSONDB::_erase(const std::string &n) +{ if (_httpAddr) { std::string body; std::map headers; @@ -142,17 +152,12 @@ void JSONDB::erase(const std::string &n) return; OSUtils::rm(path.c_str()); } - - { - Mutex::Lock _l(_db_m); - _db.erase(n); - } } -bool JSONDB::_reload(const std::string &p,const std::string &b) +void JSONDB::_reload(const std::string &p,const std::string &b) { - // Assumes _db_m is locked if (_httpAddr) { + Mutex::Lock _l(_db_m); std::string body; std::map headers; const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); @@ -161,18 +166,19 @@ bool JSONDB::_reload(const std::string &p,const std::string &b) nlohmann::json dbImg(OSUtils::jsonParse(body)); std::string tmp; if (dbImg.is_object()) { + _db.clear(); for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { if (i.value().is_object()) { tmp = i.key(); _db[tmp].obj = i.value(); } } - return true; + _ready = true; } } catch ( ... ) {} // invalid JSON, so maybe incomplete request } - return false; } else { + _ready = true; std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { @@ -181,7 +187,6 @@ bool JSONDB::_reload(const std::string &p,const std::string &b) this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); } } - return true; } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 2d3a5224..a045d1b4 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -72,29 +72,28 @@ public: template inline void filter(const std::string &prefix,F func) { - Mutex::Lock _l(_db_m); - while (!_ready) { Thread::sleep(250); - _ready = _reload(_basePath,std::string()); + _reload(_basePath,std::string()); } - - for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { - if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { - if (!func(i->first,get(i->first))) { - std::map::iterator i2(i); ++i2; - this->erase(i->first); - i = i2; - } else ++i; - } else break; + { + Mutex::Lock _l(_db_m); + for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { + if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { + if (!func(i->first,i->second.obj)) { + this->_erase(i->first); + _db.erase(i++); + } else { + ++i; + } + } else break; + } } } - inline bool operator==(const JSONDB &db) const { return ((_basePath == db._basePath)&&(_db == db._db)); } - inline bool operator!=(const JSONDB &db) const { return (!(*this == db)); } - private: - bool _reload(const std::string &p,const std::string &b); + void _erase(const std::string &n); + void _reload(const std::string &p,const std::string &b); bool _isValidObjectName(const std::string &n); std::string _genPath(const std::string &n,bool create); diff --git a/node/Node.cpp b/node/Node.cpp index 2b3f7996..ccbe9411 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -490,7 +490,8 @@ int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *d void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); - RR->localNetworkController->init(RR->identity,this); + if (networkControllerInstance) + RR->localNetworkController->init(RR->identity,this); } ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) diff --git a/osdep/Thread.hpp b/osdep/Thread.hpp index 227c2cfe..5423a8ab 100644 --- a/osdep/Thread.hpp +++ b/osdep/Thread.hpp @@ -46,7 +46,6 @@ class Thread { public: Thread() - throw() { _th = NULL; _tid = 0; @@ -54,7 +53,6 @@ public: template static inline Thread start(C *instance) - throw(std::runtime_error) { Thread t; t._th = CreateThread(NULL,0,&___zt_threadMain,(LPVOID)instance,0,&t._tid); @@ -88,7 +86,7 @@ public: CancelSynchronousIo(t._th); } - inline operator bool() const throw() { return (_th != NULL); } + inline operator bool() const { return (_th != NULL); } private: HANDLE _th; @@ -123,33 +121,18 @@ class Thread { public: Thread() - throw() { - memset(&_tid,0,sizeof(_tid)); - pthread_attr_init(&_tattr); - // This corrects for systems with abnormally small defaults (musl) and also - // shrinks the stack on systems with large defaults to save a bit of memory. - pthread_attr_setstacksize(&_tattr,ZT_THREAD_MIN_STACK_SIZE); - _started = false; - } - - ~Thread() - { - pthread_attr_destroy(&_tattr); + memset(this,0,sizeof(Thread)); } Thread(const Thread &t) - throw() { - memcpy(&_tid,&(t._tid),sizeof(_tid)); - _started = t._started; + memcpy(this,&t,sizeof(Thread)); } inline Thread &operator=(const Thread &t) - throw() { - memcpy(&_tid,&(t._tid),sizeof(_tid)); - _started = t._started; + memcpy(this,&t,sizeof(Thread)); return *this; } @@ -163,12 +146,20 @@ public: */ template static inline Thread start(C *instance) - throw(std::runtime_error) { Thread t; - t._started = true; - if (pthread_create(&t._tid,&t._tattr,&___zt_threadMain,instance)) + pthread_attr_t tattr; + pthread_attr_init(&tattr); + // This corrects for systems with abnormally small defaults (musl) and also + // shrinks the stack on systems with large defaults to save a bit of memory. + pthread_attr_setstacksize(&tattr,ZT_THREAD_MIN_STACK_SIZE); + if (pthread_create(&t._tid,&tattr,&___zt_threadMain,instance)) { + pthread_attr_destroy(&tattr); throw std::runtime_error("pthread_create() failed, unable to create thread"); + } else { + t._started = true; + pthread_attr_destroy(&tattr); + } return t; } @@ -190,11 +181,10 @@ public: */ static inline void sleep(unsigned long ms) { usleep(ms * 1000); } - inline operator bool() const throw() { return (_started); } + inline operator bool() const { return (_started); } private: pthread_t _tid; - pthread_attr_t _tattr; volatile bool _started; }; -- cgit v1.2.3 From 9e80db0fd169de19d5d343e8b6998c8ace4aeb22 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 27 Apr 2017 00:59:36 -0700 Subject: Cleanup, fix a valgrind error, stack use reduction. --- controller/EmbeddedNetworkController.cpp | 134 +- controller/EmbeddedNetworkController.hpp | 3 +- controller/JSONDB.cpp | 57 +- controller/JSONDB.hpp | 55 +- ext/json/LICENSE.MIT | 17 +- ext/json/README.md | 420 +- ext/json/json.hpp | 6976 +++++++++++++++++++----------- make-linux.mk | 9 +- node/CertificateOfOwnership.hpp | 2 + root-watcher/schema.sql | 3 +- selftest.cpp | 47 - 11 files changed, 4910 insertions(+), 2813 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index be53f2b8..a41a4a94 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2015 ZeroTier, Inc-> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,9 +30,9 @@ #include #include #include -#include #include #include +#include #include "../include/ZeroTierOne.h" #include "../node/Constants.hpp" @@ -1017,7 +1017,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - if (network != origNetwork) { + if (true) { json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["lastModified"] = now; @@ -1235,8 +1235,9 @@ void EmbeddedNetworkController::_request( } // These are always the same, but make sure they are set - member["id"] = identity.address().toString(); - member["address"] = member["id"]; + const std::string addrs(identity.address().toString()); + member["id"] = addrs; + member["address"] = addrs; member["nwid"] = nwids; // Determine whether and how member is authorized @@ -1356,8 +1357,6 @@ void EmbeddedNetworkController::_request( // If we made it this far, they are authorized. // ------------------------------------------------------------------------- - NetworkConfig nc; - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; if (now > ns.mostRecentDeauthTime) { // If we recently de-authorized a member, shrink credential TTL/max delta to @@ -1371,19 +1370,21 @@ void EmbeddedNetworkController::_request( } } - nc.networkId = nwid; - nc.type = OSUtils::jsonBool(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.credentialTimeMaxDelta = credentialtmd; - nc.revision = OSUtils::jsonInt(network["revision"],0ULL); - nc.issuedTo = identity.address(); - if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),OSUtils::jsonString(network["name"],"").c_str()); - nc.multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); + std::auto_ptr nc(new NetworkConfig()); + + nc->networkId = nwid; + nc->type = OSUtils::jsonBool(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc->timestamp = now; + nc->credentialTimeMaxDelta = credentialtmd; + nc->revision = OSUtils::jsonInt(network["revision"],0ULL); + nc->issuedTo = identity.address(); + if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc->name,sizeof(nc->name),OSUtils::jsonString(network["name"],"").c_str()); + nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) - nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); + nc->addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -1399,15 +1400,15 @@ void EmbeddedNetworkController::_request( // Old versions with no rules engine support get an allow everything rule. // Since rules are enforced bidirectionally, newer versions *will* still // enforce rules on the inbound side. - nc.ruleCount = 1; - nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; + nc->ruleCount = 1; + nc->rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; } else { if (rules.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + if (nc->ruleCount >= ZT_MAX_NETWORK_RULES) break; - if (_parseRule(rules[i],nc.rules[nc.ruleCount])) - ++nc.ruleCount; + if (_parseRule(rules[i],nc->rules[nc->ruleCount])) + ++nc->ruleCount; } } @@ -1451,10 +1452,10 @@ void EmbeddedNetworkController::_request( ++caprc; } } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) - ++nc.capabilityCount; - if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + nc->capabilities[nc->capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc->capabilities[nc->capabilityCount].sign(_signingId,identity.address())) + ++nc->capabilityCount; + if (nc->capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) break; } } @@ -1485,17 +1486,17 @@ void EmbeddedNetworkController::_request( } } for(std::map< uint32_t,uint32_t >::const_iterator t(memberTagsById.begin());t!=memberTagsById.end();++t) { - if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + if (nc->tagCount >= ZT_MAX_NETWORK_TAGS) break; - nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(_signingId)) - ++nc.tagCount; + nc->tags[nc->tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); + if (nc->tags[nc->tagCount].sign(_signingId)) + ++nc->tagCount; } } if (routes.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + if (nc->routeCount >= ZT_MAX_NETWORK_ROUTES) break; json &route = routes[i]; json &target = route["target"]; @@ -1505,11 +1506,11 @@ void EmbeddedNetworkController::_request( InetAddress v; if (via.is_string()) v.fromString(via.get()); if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + ZT_VirtualNetworkRoute *r = &(nc->routes[nc->routeCount]); *(reinterpret_cast(&(r->target))) = t; if (v.ss_family == t.ss_family) *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; + ++nc->routeCount; } } } @@ -1518,13 +1519,13 @@ void EmbeddedNetworkController::_request( const bool noAutoAssignIps = OSUtils::jsonBool(member["noAutoAssignIps"],false); if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { - if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc->staticIps[nc->staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); + nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc->staticIps[nc->staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); + nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } } @@ -1542,15 +1543,15 @@ void EmbeddedNetworkController::_request( // this route, ignoring the netmask bits field of the assigned IP itself. Using that was worthless and a source // of user error / poor UX. int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + for(unsigned int rk=0;rkrouteCount;++rk) { + if ( (!nc->routes[rk].via.ss_family) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip)) ) + routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } if (routedNetmaskBits > 0) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { ip.setPort(routedNetmaskBits); - nc.staticIps[nc.staticIpCount++] = ip; + nc->staticIps[nc->staticIpCount++] = ip; } if (ip.ss_family == AF_INET) haveManagedIpv4AutoAssignment = true; @@ -1601,9 +1602,9 @@ void EmbeddedNetworkController::_request( // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + for(unsigned int rk=0;rkrouteCount;++rk) { + if ( (!nc->routes[rk].via.ss_family) && (nc->routes[rk].target.ss_family == AF_INET6) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } // If it's routed, then try to claim and assign it and if successful end loop @@ -1611,8 +1612,8 @@ void EmbeddedNetworkController::_request( ipAssignments.push_back(ip6.toIpString()); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc->staticIps[nc->staticIpCount++] = ip6; haveManagedIpv6AutoAssignment = true; break; } @@ -1646,10 +1647,10 @@ void EmbeddedNetworkController::_request( // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = -1; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + for(unsigned int rk=0;rkrouteCount;++rk) { + if (nc->routes[rk].target.ss_family == AF_INET) { + uint32_t targetIp = Utils::ntoh((uint32_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_port)); if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { routedNetmaskBits = targetBits; break; @@ -1662,8 +1663,8 @@ void EmbeddedNetworkController::_request( if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip4)) ) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); v4ip->sin_family = AF_INET; v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); v4ip->sin_addr.s_addr = Utils::hton(ip); @@ -1678,17 +1679,17 @@ void EmbeddedNetworkController::_request( } // Issue a certificate of ownership for all static IPs - if (nc.staticIpCount) { - nc.certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); - for(unsigned int i=0;istaticIpCount) { + nc->certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); + for(unsigned int i=0;istaticIpCount;++i) + nc->certificatesOfOwnership[0].addThing(nc->staticIps[i]); + nc->certificatesOfOwnership[0].sign(_signingId); + nc->certificateOfOwnershipCount = 1; } CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); if (com.sign(_signingId)) { - nc.com = com; + nc->com = com; } else { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); return; @@ -1699,7 +1700,7 @@ void EmbeddedNetworkController::_request( _db.saveNetworkMember(nwid,identity.address().toInt(),member); } - _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); + _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) @@ -1716,11 +1717,10 @@ void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,con online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); } if (online) { - Dictionary *metaData = new Dictionary(mdstr.c_str()); + Dictionary metaData(mdstr.c_str()); try { - this->request(nwid,InetAddress(),0,id,*metaData); + this->request(nwid,InetAddress(),0,id,metaData); } catch ( ... ) {} - delete metaData; } } } catch ( ... ) {} diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 4dada88e..8a220139 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -34,6 +34,7 @@ #include "../node/Utils.hpp" #include "../node/Address.hpp" #include "../node/InetAddress.hpp" +#include "../node/NonCopyable.hpp" #include "../osdep/OSUtils.hpp" #include "../osdep/Thread.hpp" @@ -50,7 +51,7 @@ namespace ZeroTier { class Node; -class EmbeddedNetworkController : public NetworkController +class EmbeddedNetworkController : public NetworkController,NonCopyable { public: /** diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index a2fe7f2a..01eb50cd 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -107,9 +107,64 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) } } +bool JSONDB::hasNetwork(const uint64_t networkId) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + return (i != _networks.end()); +} + +bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + config = i->second.config; + return true; +} + +bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + ns = i->second.summaryInfo; + return true; +} + +int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return 0; + std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return 1; + networkConfig = i->second.config; + memberConfig = j->second; + ns = i->second.summaryInfo; + return 3; +} + +bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const +{ + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return false; + memberConfig = j->second; + return true; +} + void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { - char n[256]; + char n[64]; Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig)); { diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index f89ff6c9..8db5cc48 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -63,63 +63,18 @@ public: bool writeRaw(const std::string &n,const std::string &obj); - inline bool hasNetwork(const uint64_t networkId) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - return (i != _networks.end()); - } + bool hasNetwork(const uint64_t networkId) const; - inline bool getNetwork(const uint64_t networkId,nlohmann::json &config) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - config = i->second.config; - return true; - } + bool getNetwork(const uint64_t networkId,nlohmann::json &config) const; - inline bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - ns = i->second.summaryInfo; - return true; - } + bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const; /** * @return Bit mask: 0 == none, 1 == network only, 3 == network and member */ - inline int getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return 0; - networkConfig = i->second.config; - ns = i->second.summaryInfo; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); - if (j == i->second.members.end()) - return 1; - memberConfig = j->second; - return 3; - } + int getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const; - inline bool getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const - { - Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - if (i == _networks.end()) - return false; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); - if (j == i->second.members.end()) - return false; - memberConfig = j->second; - return true; - } + bool getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const; void saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig); diff --git a/ext/json/LICENSE.MIT b/ext/json/LICENSE.MIT index e2ac4891..00599afe 100644 --- a/ext/json/LICENSE.MIT +++ b/ext/json/LICENSE.MIT @@ -1,14 +1,13 @@ -The library is licensed under the MIT License -: +MIT License -Copyright (c) 2013-2016 Niels Lohmann +Copyright (c) 2013-2017 Niels Lohmann -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. diff --git a/ext/json/README.md b/ext/json/README.md index 4bcbe97f..fc6dde9b 100644 --- a/ext/json/README.md +++ b/ext/json/README.md @@ -3,13 +3,34 @@ [![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) [![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk/branch/develop?svg=true)](https://ci.appveyor.com/project/nlohmann/json) [![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) -[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/fsf5FqYe6GoX68W6) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/f3732b3327e34358a0e9d1fe9f661f08)](https://www.codacy.com/app/nlohmann/json?utm_source=github.com&utm_medium=referral&utm_content=nlohmann/json&utm_campaign=Badge_Grade) +[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/nv9fOg0XVVhWmFFy) [![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) [![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) [![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/289/badge)](https://bestpractices.coreinfrastructure.org/projects/289) +- [Design goals](#design-goals) +- [Integration](#integration) +- [Examples](#examples) + - [JSON as first-class data type](#json-as-first-class-data-type) + - [Serialization / Deserialization](#serialization--deserialization) + - [STL-like access](#stl-like-access) + - [Conversion from STL containers](#conversion-from-stl-containers) + - [JSON Pointer and JSON Patch](#json-pointer-and-json-patch) + - [Implicit conversions](#implicit-conversions) + - [Conversions to/from arbitrary types](#arbitrary-types-conversions) + - [Binary formats (CBOR and MessagePack)](#binary-formats-cbor-and-messagepack) +- [Supported compilers](#supported-compilers) +- [License](#license) +- [Thanks](#thanks) +- [Used third-party tools](#used-third-party-tools) +- [Projects using JSON for Modern C++](#projects-using-json-for-modern-c) +- [Notes](#notes) +- [Execute unit tests](#execute-unit-tests) + ## Design goals There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: @@ -24,7 +45,7 @@ Other aspects were not so important to us: - **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. -- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). +- **Speed**. There are certainly [faster JSON libraries](https://github.com/miloyip/nativejson-benchmark#parsing-time) out there. However, if your goal is to speed up your development by adding JSON support with a single header, then this library is the way to go. If you know how to use a `std::vector` or `std::map`, you are already set. See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. @@ -44,9 +65,15 @@ to the files you want to use JSON objects. That's it. Do not forget to set the n :beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. +:warning: [Version 3.0.0](https://github.com/nlohmann/json/wiki/Road-toward-3.0.0) is currently under development. Branch `develop` is used for the ongoing work and is probably **unstable**. Please use the `master` branch for the last stable version 2.1.1. + ## Examples +Beside the examples below, you may want to check the [documentation](https://nlohmann.github.io/json/) where each function contains a separate code example (e.g., check out [`emplace()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a602f275f0359ab181221384989810604.html#a602f275f0359ab181221384989810604)). All [example files](https://github.com/nlohmann/json/tree/develop/doc/examples) can be compiled and executed on their own (e.g., file [emplace.cpp](https://github.com/nlohmann/json/blob/develop/doc/examples/emplace.cpp)). + +### JSON as first-class data type + Here are some examples to give you an idea how to use the class. Assume you want to create the JSON object @@ -129,6 +156,8 @@ json array_not_object = { json::array({"currency", "USD"}), json::array({"value" ### Serialization / Deserialization +#### To/from strings + You can create an object (deserialization) by appending `_json` to a string literal: ```cpp @@ -142,8 +171,14 @@ auto j2 = R"( "pi": 3.141 } )"_json; +``` + +Note that without appending the `_json` suffix, the passed string literal is not parsed, but just used as JSON string value. That is, `json j = "{ \"happy\": true, \"pi\": 3.141 }"` would just store the string `"{ "happy": true, "pi": 3.141 }"` rather than parsing the actual object. -// or explicitly +The above example can also be expressed explicitly using `json::parse()`: + +```cpp +// parse explicitly auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); ``` @@ -162,6 +197,8 @@ std::cout << j.dump(4) << std::endl; // } ``` +#### To/from streams (e.g. files, string streams) + You can also use streams to serialize and deserialize: ```cpp @@ -176,10 +213,37 @@ std::cout << j; std::cout << std::setw(4) << j << std::endl; ``` -These operators work for any subclasses of `std::istream` or `std::ostream`. +These operators work for any subclasses of `std::istream` or `std::ostream`. Here is the same example with files: + +```cpp +// read a JSON file +std::ifstream i("file.json"); +json j; +i >> j; + +// write prettified JSON to another file +std::ofstream o("pretty.json"); +o << std::setw(4) << j << std::endl; +``` Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. +#### Read from iterator range + +You can also read JSON from an iterator range; that is, from any container accessible by iterators whose content is stored as contiguous byte sequence, for instance a `std::vector`: + +```cpp +std::vector v = {'t', 'r', 'u', 'e'}; +json j = json::parse(v.begin(), v.end()); +``` + +You may leave the iterators for the range [begin, end): + +```cpp +std::vector v = {'t', 'r', 'u', 'e'}; +json j = json::parse(v); +``` + ### STL-like access @@ -192,6 +256,9 @@ j.push_back("foo"); j.push_back(1); j.push_back(true); +// also use emplace_back +j.emplace_back(1.78); + // iterate the array for (json::iterator it = j.begin(); it != j.end(); ++it) { std::cout << *it << '\n'; @@ -207,6 +274,9 @@ const std::string tmp = j[0]; j[1] = 42; bool foo = j.at(2); +// comparison +j == "[\"foo\", 1, true]"_json; // true + // other stuff j.size(); // 3 entries j.empty(); // false @@ -221,15 +291,15 @@ j.is_object(); j.is_array(); j.is_string(); -// comparison -j == "[\"foo\", 1, true]"_json; // true - // create an object json o; o["foo"] = 23; o["bar"] = false; o["baz"] = 3.141; +// also use emplace +o.emplace("weather", "sunny"); + // special iterator member functions for objects for (json::iterator it = o.begin(); it != o.end(); ++it) { std::cout << it.key() << " : " << it.value() << "\n"; @@ -383,6 +453,252 @@ int vi = jn.get(); // etc. ``` +### Arbitrary types conversions + +Every type can be serialized in JSON, not just STL-containers and scalar types. Usually, you would do something along those lines: + +```cpp +namespace ns { + // a simple struct to model a person + struct person { + std::string name; + std::string address; + int age; + }; +} + +ns::person p = {"Ned Flanders", "744 Evergreen Terrace", 60}; + +// convert to JSON: copy each value into the JSON object +json j; +j["name"] = p.name; +j["address"] = p.address; +j["age"] = p.age; + +// ... + +// convert from JSON: copy each value from the JSON object +ns::person p { + j["name"].get(), + j["address"].get(), + j["age"].get() +}; +``` + +It works, but that's quite a lot of boilerplate... Fortunately, there's a better way: + +```cpp +// create a person +ns::person p {"Ned Flanders", "744 Evergreen Terrace", 60}; + +// conversion: person -> json +json j = p; + +std::cout << j << std::endl; +// {"address":"744 Evergreen Terrace","age":60,"name":"Ned Flanders"} + +// conversion: json -> person +ns::person p2 = j; + +// that's it +assert(p == p2); +``` + +#### Basic usage + +To make this work with one of your types, you only need to provide two functions: + +```cpp +using nlohmann::json; + +namespace ns { + void to_json(json& j, const person& p) { + j = json{{"name", p.name}, {"address", p.address}, {"age", p.age}}; + } + + void from_json(const json& j, person& p) { + p.name = j.at("name").get(); + p.address = j.at("address").get(); + p.age = j.at("age").get(); + } +} // namespace ns +``` + +That's all! When calling the `json` constructor with your type, your custom `to_json` method will be automatically called. +Likewise, when calling `get()`, the `from_json` method will be called. + +Some important things: + +* Those methods **MUST** be in your type's namespace (which can be the global namespace), or the library will not be able to locate them (in this example, they are in namespace `ns`, where `person` is defined). +* When using `get()`, `your_type` **MUST** be [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). (There is a way to bypass this requirement described later.) +* In function `from_json`, use function [`at()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a93403e803947b86f4da2d1fb3345cf2c.html#a93403e803947b86f4da2d1fb3345cf2c) to access the object values rather than `operator[]`. In case a key does not exists, `at` throws an exception that you can handle, whereas `operator[]` exhibits undefined behavior. + +#### How do I convert third-party types? + +This requires a bit more advanced technique. But first, let's see how this conversion mechanism works: + +The library uses **JSON Serializers** to convert types to json. +The default serializer for `nlohmann::json` is `nlohmann::adl_serializer` (ADL means [Argument-Dependent Lookup](http://en.cppreference.com/w/cpp/language/adl)). + +It is implemented like this (simplified): + +```cpp +template +struct adl_serializer { + static void to_json(json& j, const T& value) { + // calls the "to_json" method in T's namespace + } + + static void from_json(const json& j, T& value) { + // same thing, but with the "from_json" method + } +}; +``` + +This serializer works fine when you have control over the type's namespace. However, what about `boost::optional`, or `std::filesystem::path` (C++17)? Hijacking the `boost` namespace is pretty bad, and it's illegal to add something other than template specializations to `std`... + +To solve this, you need to add a specialization of `adl_serializer` to the `nlohmann` namespace, here's an example: + +```cpp +// partial specialization (full specialization works too) +namespace nlohmann { + template + struct adl_serializer> { + static void to_json(json& j, const boost::optional& opt) { + if (opt == boost::none) { + j = nullptr; + } else { + j = *opt; // this will call adl_serializer::to_json which will + // find the free function to_json in T's namespace! + } + } + + static void from_json(const json& j, boost::optional& opt) { + if (j.is_null()) { + opt = boost::none; + } else { + opt = j.get(); // same as above, but with + // adl_serializer::from_json + } + } + }; +} +``` + +#### How can I use `get()` for non-default constructible/non-copyable types? + +There is a way, if your type is [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible). You will need to specialize the `adl_serializer` as well, but with a special `from_json` overload: + +```cpp +struct move_only_type { + move_only_type() = delete; + move_only_type(int ii): i(ii) {} + move_only_type(const move_only_type&) = delete; + move_only_type(move_only_type&&) = default; + + int i; +}; + +namespace nlohmann { + template <> + struct adl_serializer { + // note: the return type is no longer 'void', and the method only takes + // one argument + static move_only_type from_json(const json& j) { + return {j.get()}; + } + + // Here's the catch! You must provide a to_json method! Otherwise you + // will not be able to convert move_only_type to json, since you fully + // specialized adl_serializer on that type + static void to_json(json& j, move_only_type t) { + j = t.i; + } + }; +} +``` + +#### Can I write my own serializer? (Advanced use) + +Yes. You might want to take a look at [`unit-udt.cpp`](https://github.com/nlohmann/json/blob/develop/test/src/unit-udt.cpp) in the test suite, to see a few examples. + +If you write your own serializer, you'll need to do a few things: + +* use a different `basic_json` alias than `nlohmann::json` (the last template parameter of `basic_json` is the `JSONSerializer`) +* use your `basic_json` alias (or a template parameter) in all your `to_json`/`from_json` methods +* use `nlohmann::to_json` and `nlohmann::from_json` when you need ADL + +Here is an example, without simplifications, that only accepts types with a size <= 32, and uses ADL. + +```cpp +// You should use void as a second template argument +// if you don't need compile-time checks on T +template::type> +struct less_than_32_serializer { + template + static void to_json(BasicJsonType& j, T value) { + // we want to use ADL, and call the correct to_json overload + using nlohmann::to_json; // this method is called by adl_serializer, + // this is where the magic happens + to_json(j, value); + } + + template + static void from_json(const BasicJsonType& j, T& value) { + // same thing here + using nlohmann::from_json; + from_json(j, value); + } +}; +``` + +Be **very** careful when reimplementing your serializer, you can stack overflow if you don't pay attention: + +```cpp +template +struct bad_serializer +{ + template + static void to_json(BasicJsonType& j, const T& value) { + // this calls BasicJsonType::json_serializer::to_json(j, value); + // if BasicJsonType::json_serializer == bad_serializer ... oops! + j = value; + } + + template + static void to_json(const BasicJsonType& j, T& value) { + // this calls BasicJsonType::json_serializer::from_json(j, value); + // if BasicJsonType::json_serializer == bad_serializer ... oops! + value = j.template get(); // oops! + } +}; +``` + +### Binary formats (CBOR and MessagePack) + +Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [CBOR](http://cbor.io) (Concise Binary Object Representation) and [MessagePack](http://msgpack.org) to efficiently encode JSON values to byte vectors and to decode such vectors. + +```cpp +// create a JSON value +json j = R"({"compact": true, "schema": 0})"_json; + +// serialize to CBOR +std::vector v_cbor = json::to_cbor(j); + +// 0xa2, 0x67, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xf5, 0x66, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 + +// roundtrip +json j_from_cbor = json::from_cbor(v_cbor); + +// serialize to MessagePack +std::vector v_msgpack = json::to_msgpack(j); + +// 0x82, 0xa7, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xc3, 0xa6, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 + +// roundtrip +json j_from_msgpack = json::from_msgpack(v_msgpack); +``` + ## Supported compilers @@ -391,6 +707,7 @@ Though it's 2016 already, the support for C++11 is still a bit sparse. Currently - GCC 4.9 - 6.0 (and possibly later) - Clang 3.4 - 3.9 (and possibly later) - Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) +- Microsoft Visual C++ 2017 / Build Tools 15.1.548.43366 (and possibly later) I would be happy to learn about other compilers/versions. @@ -423,16 +740,14 @@ The following compilers are currently used in continuous integration at [Travis] | Clang 3.7.1 | Ubuntu 14.04.4 LTS | clang version 3.7.1 (tags/RELEASE_371/final) | | Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | | Clang 3.8.1 | Ubuntu 14.04.4 LTS | clang version 3.8.1 (tags/RELEASE_381/final) | -| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | -| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | -| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | | Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | -| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | | Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 (OSX 10.11.6) | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.1 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.2 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.3 | Darwin Kernel Version 16.5.0 (macOS 10.12.4) | Apple LLVM version 8.1.0 (clang-802.0.38) | | Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | - +| Visual Studio 2017 | Windows Server 2016 | Microsoft (R) Build Engine version 15.1.548.43366 | ## License @@ -440,7 +755,7 @@ The following compilers are currently used in continuous integration at [Travis] The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): -Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) +Copyright © 2013-2017 [Niels Lohmann](http://nlohmann.me) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: @@ -465,7 +780,7 @@ I deeply appreciate the help of the following people. - [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. - [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. - [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. -- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. +- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. In particular, he pushed forward the implementation of user-defined types. - [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. - [dariomt](https://github.com/dariomt) fixed some typos in the examples. - [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. @@ -493,14 +808,70 @@ I deeply appreciate the help of the following people. - [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. - [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - [Thomas Braun](https://github.com/t-b) fixed a warning in a test case. -- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). +- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). He also implemented the magic behind the serialization/deserialization of user-defined types. - [Stefan](https://github.com/5tefan) fixed a minor issue in the documentation. - [Vasil Dimov](https://github.com/vasild) fixed the documentation regarding conversions from `std::multiset`. - [ChristophJud](https://github.com/ChristophJud) overworked the CMake files to ease project inclusion. -- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable. +- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable and added Visual Studio 17 to the build matrix. - [Denis Andrejew](https://github.com/seeekr) fixed a grammar issue in the README file. - -Thanks a lot for helping out! +- [Pierre-Antoine Lacaze](https://github.com/palacaze) found a subtle bug in the `dump()` function. +- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](http://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling, found some nice performance improvements in the parser, improved the benchmarking code, and realized locale-independent number parsing and printing. +- [cgzones](https://github.com/cgzones) had an idea how to fix the Coverity scan. +- [Jared Grubb](https://github.com/jaredgrubb) silenced a nasty documentation warning. +- [Yixin Zhang](https://github.com/qwename) fixed an integer overflow check. +- [Bosswestfalen](https://github.com/Bosswestfalen) merged two iterator classes into a smaller one. +- [Daniel599](https://github.com/Daniel599) helped to get Travis execute the tests with Clang's sanitizers. +- [Jonathan Lee](https://github.com/vjon) fixed an example in the README file. +- [gnzlbg](https://github.com/gnzlbg) supported the implementation of user-defined types. +- [Alexej Harm](https://github.com/qis) helped to get the user-defined types working with Visual Studio. +- [Jared Grubb](https://github.com/jaredgrubb) supported the implementation of user-defined types. +- [EnricoBilla](https://github.com/EnricoBilla) noted a typo in an example. +- [Martin Hořeňovský](https://github.com/horenmar) found a way for a 2x speedup for the compilation time of the test suite. +- [ukhegg](https://github.com/ukhegg) found proposed an improvement for the examples section. +- [rswanson-ihi](https://github.com/rswanson-ihi) noted a typo in the README. +- [Mihai Stan](https://github.com/stanmihai4) fixed a bug in the comparison with `nullptr`s. +- [Tushar Maheshwari](https://github.com/tusharpm) added [cotire](https://github.com/sakra/cotire) support to speed up the compilation. +- [TedLyngmo](https://github.com/TedLyngmo) noted a typo in the README, removed unnecessary bit arithmetic, and fixed some `-Weffc++` warnings. +- [Krzysztof Woś](https://github.com/krzysztofwos) made exceptions more visible. +- [ftillier](https://github.com/ftillier) fixed a compiler warning. +- [tinloaf](https://github.com/tinloaf) made sure all pushed warnings are properly popped. +- [Fytch](https://github.com/Fytch) found a bug in the documentation. + +Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone. + + +## Used third-party tools + +The library itself contains of a single header file licensed under the MIT license. However, it is built, tested, documented, and whatnot using a lot of third-party tools and services. Thanks a lot! + +- [**American fuzzy lop**](http://lcamtuf.coredump.cx/afl/) for fuzz testing +- [**AppVeyor**](https://www.appveyor.com) for [continuous integration](https://ci.appveyor.com/project/nlohmann/json) on Windows +- [**Artistic Style**](http://astyle.sourceforge.net) for automatic source code identation +- [**benchpress**](https://github.com/sbs-ableton/benchpress) to benchmark the code +- [**Catch**](https://github.com/philsquared/Catch) for the unit tests +- [**Clang**](http://clang.llvm.org) for compilation with code sanitizers +- [**Cmake**](https://cmake.org) for build automation +- [**Codacity**](https://www.codacy.com) for further [code analysis](https://www.codacy.com/app/nlohmann/json) +- [**cotire**](https://github.com/sakra/cotire) to speed of compilation +- [**Coveralls**](https://coveralls.io) to measure [code coverage](https://coveralls.io/github/nlohmann/json) +- [**Coverity Scan**](https://scan.coverity.com) for [static analysis](https://scan.coverity.com/projects/nlohmann-json) +- [**cppcheck**](http://cppcheck.sourceforge.net) for static analysis +- [**cxxopts**](https://github.com/jarro2783/cxxopts) to let benchpress parse command-line parameters +- [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/) +- [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages +- [**Github Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) +- [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz +- [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library +- [**re2c**](http://re2c.org) to generate an automaton for the lexical analysis +- [**send_to_wandbox**](https://github.com/nlohmann/json/blob/develop/doc/scripts/send_to_wandbox.py) to send code examples to [Wandbox](http://melpon.org/wandbox) +- [**Travis**](https://travis-ci.org) for [continuous integration](https://travis-ci.org/nlohmann/json) on Linux and macOS +- [**Valgrind**](http://valgrind.org) to check for correct memory management +- [**Wandbox**](http://melpon.org/wandbox) for [online examples](http://melpon.org/wandbox/permlink/4NEU6ZZMoM9lpIex) + + +## Projects using JSON for Modern C++ + +The library is currently used in Apple macOS Sierra and iOS 10. I am not sure what they are using the library for, but I am happy that it runs on so many devices. ## Notes @@ -512,6 +883,10 @@ Thanks a lot for helping out! - Other encodings such as Latin-1, UTF-16, or UTF-32 are not supported and will yield parse errors. - [Unicode noncharacters](http://www.unicode.org/faq/private_use.html#nonchar1) will not be replaced by the library. - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors. + - The strings stored in the library are UTF-8 encoded. When using the default string type (`std::string`), note that its length/size functions return the number of stored bytes rather than the number of characters or glyphs. +- The code can be compiled without C++ **runtime type identification** features; that is, you can use the `-fno-rtti` compiler flag. +- **Exceptions** are used widely within the library. They can, however, be switched off with either using the compiler flag `-fno-exceptions` or by defining the symbol `JSON_NOEXCEPTION`. In this case, exceptions are replaced by an `abort()` call. +- By default, the library does not preserve the **insertion order of object elements**. This is standards-compliant, as the [JSON standard](https://tools.ietf.org/html/rfc7159.html) defines objects as "an unordered collection of zero or more name/value pairs". If you do want to preserve the insertion order, you can specialize the object type with containers like [`tsl::ordered_map`](https://github.com/Tessil/ordered-map) or [`nlohmann::fifo_map`](https://github.com/nlohmann/fifo_map). ## Execute unit tests @@ -519,10 +894,11 @@ Thanks a lot for helping out! To compile and run the tests, you need to execute ```sh -$ make check +$ make json_unit -Ctest +$ ./test/json_unit "*" =============================================================================== -All tests passed (8905491 assertions in 36 test cases) +All tests passed (11203022 assertions in 48 test cases) ``` Alternatively, you can use [CMake](https://cmake.org) and run diff --git a/ext/json/json.hpp b/ext/json/json.hpp index 9d48e7a6..8a8b876a 100644 --- a/ext/json/json.hpp +++ b/ext/json/json.hpp @@ -1,7 +1,7 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.0.10 +| | |__ | | | | | | version 2.1.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . @@ -29,42 +29,39 @@ SOFTWARE. #ifndef NLOHMANN_JSON_HPP #define NLOHMANN_JSON_HPP -#include // all_of, for_each, transform +#include // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform #include // array #include // assert -#include // isdigit #include // and, not, or -#include // isfinite, ldexp, signbit +#include // lconv, localeconv +#include // isfinite, labs, ldexp, signbit #include // nullptr_t, ptrdiff_t, size_t #include // int64_t, uint64_t -#include // strtod, strtof, strtold, strtoul +#include // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull #include // strlen +#include // forward_list #include // function, hash, less #include // initializer_list -#include // setw #include // istream, ostream -#include // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator +#include // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator #include // numeric_limits #include // locale #include // map #include // addressof, allocator, allocator_traits, unique_ptr #include // accumulate #include // stringstream -#include // domain_error, invalid_argument, out_of_range #include // getline, stoi, string, to_string -#include // add_pointer, enable_if, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_floating_point, is_integral, is_nothrow_move_assignable, std::is_nothrow_move_constructible, std::is_pointer, std::is_reference, std::is_same, remove_const, remove_pointer, remove_reference +#include // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type #include // declval, forward, make_pair, move, pair, swap #include // vector // exclude unsupported compilers #if defined(__clang__) - #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) - #if CLANG_VERSION < 30400 + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" #endif #elif defined(__GNUC__) - #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - #if GCC_VERSION < 40900 + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" #endif #endif @@ -90,6 +87,17 @@ SOFTWARE. #define JSON_DEPRECATED #endif +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) +#endif + /*! @brief namespace for Niels Lohmann @see https://github.com/nlohmann @@ -98,1466 +106,2175 @@ SOFTWARE. namespace nlohmann { - /*! @brief unnamed namespace with internal helper functions -@since version 1.0.0 + +This namespace collects some functions that could not be defined inside the +@ref basic_json class. + +@since version 2.1.0 */ -namespace +namespace detail { +//////////////// +// exceptions // +//////////////// + /*! -@brief Helper to determine whether there's a key_type for T. +@brief general exception of the @ref basic_json class -Thus helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. +Extension of std::exception objects with a member @a id for exception ids. -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0, overworked in version 2.0.6 +@note To have nothrow-copy-constructible exceptions, we internally use + std::runtime_error which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. + +@since version 3.0.0 */ -template -struct has_mapped_type +class exception : public std::exception { + public: + /// returns the explanatory string + virtual const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) + : id(id_), m(what_arg) + {} + + static std::string name(const std::string& ename, int id) + { + return "[json.exception." + ename + "." + std::to_string(id) + "] "; + } + private: - template - static int detect(U&&); + /// an exception object as storage for error messages + std::runtime_error m; +}; - static void detect(...); +/*! +@brief exception indicating a parse error + +This excpetion is thrown by the library when a parse error occurs. Parse +errors can occur during the deserialization of JSON text as well as when +using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +@note For an input with n bytes, 1 is the index of the first character + and n+1 is the index of the terminating null byte or the end of + file. This also holds true when reading a byte vector (CBOR or + MessagePack). + +Exceptions have ids 1xx. + +name / id | example massage | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.111 | parse error: bad input stream | Parsing CBOR or MessagePack from an input stream where the [`badbit` or `failbit`](http://en.cppreference.com/w/cpp/io/ios_base/iostate) is set. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. + +@since version 3.0.0 +*/ +class parse_error : public exception +{ public: - static constexpr bool value = - std::is_integral()))>::value; -}; + /*! + @brief create a parse error exception + @param[in] id the id of the exception + @param[in] byte_ the byte index where the error occured (or 0 if + the position cannot be determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id, size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id) + "parse error" + + (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id, byte_, w.c_str()); + } -} + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character + and n+1 is the index of the terminating null byte or the end of + file. This also holds true when reading a byte vector (CBOR or + MessagePack). + */ + const size_t byte; + + private: + parse_error(int id_, size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) + {} +}; /*! -@brief a class to store JSON values +@brief exception indicating errors with iterators + +Exceptions have ids 2xx. + +name / id | example massage | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compated, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id) + what_arg; + return invalid_iterator(id, w.c_str()); + } -@tparam ObjectType type for JSON objects (`std::map` by default; will be used -in @ref object_t) -@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used -in @ref array_t) -@tparam StringType type for JSON strings and object keys (`std::string` by -default; will be used in @ref string_t) -@tparam BooleanType type for JSON booleans (`bool` by default; will be used -in @ref boolean_t) -@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by -default; will be used in @ref number_integer_t) -@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c -`uint64_t` by default; will be used in @ref number_unsigned_t) -@tparam NumberFloatType type for JSON floating-point numbers (`double` by -default; will be used in @ref number_float_t) -@tparam AllocatorType type of the allocator to use (`std::allocator` by -default) + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; -@requirement The class satisfies the following concept requirements: -- Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): - JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): - A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): - A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): - A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): - A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): - JSON values can be destructed. -- Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): - JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): - All non-static data members are private and standard layout types, the class - has no virtual functions or (virtual) base classes. -- Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): - JSON values can be compared with `==`, see @ref - operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): - JSON values can be compared with `<`, see @ref - operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): - Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of - other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): - JSON values can be compared against `std::nullptr_t` objects which are used - to model the `null` value. -- Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): - JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); - JSON values can be used like STL containers and provide reverse iterator - access. +/*! +@brief exception indicating executing a member function with a wrong type + +Exceptions have ids 3xx. + +name / id | example massage | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id, const std::string& what_arg) + { + std::string w = exception::name("type_error", id) + what_arg; + return type_error(id, w.c_str()); + } -@invariant The member variables @a m_value and @a m_type have the following -relationship: -- If `m_type == value_t::object`, then `m_value.object != nullptr`. -- If `m_type == value_t::array`, then `m_value.array != nullptr`. -- If `m_type == value_t::string`, then `m_value.string != nullptr`. -The invariants are checked by member function assert_invariant(). + private: + type_error(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; -@internal -@note ObjectType trick from http://stackoverflow.com/a/9860911 -@endinternal +/*! +@brief exception indicating access out of the defined range -@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange -Format](http://rfc7159.net/rfc7159) +Exceptions have ids 4xx. -@since version 1.0.0 +name / id | example massage | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -@nosubgrouping +@since version 3.0.0 */ -template < - template class ObjectType = std::map, - template class ArrayType = std::vector, - class StringType = std::string, - class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator - > -class basic_json +class out_of_range : public exception { + public: + static out_of_range create(int id, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id) + what_arg; + return out_of_range(id, w.c_str()); + } + private: - /// workaround type for MSVC - using basic_json_t = basic_json; + out_of_range(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; - public: - // forward declarations - template class iter_impl; - template class json_reverse_iterator; - class json_pointer; +/*! +@brief exception indicating other errors - ///////////////////// - // container types // - ///////////////////// +Exceptions have ids 5xx. - /// @name container types - /// The canonic container types to use @ref basic_json like any other STL - /// container. - /// @{ +name / id | example massage | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. - /// the type of elements in a basic_json container - using value_type = basic_json; +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id, const std::string& what_arg) + { + std::string w = exception::name("other_error", id) + what_arg; + return other_error(id, w.c_str()); + } - /// the type of an element reference - using reference = value_type&; - /// the type of an element const reference - using const_reference = const value_type&; + private: + other_error(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; - /// a type to represent differences between iterators - using difference_type = std::ptrdiff_t; - /// a type to represent container sizes - using size_type = std::size_t; - /// the allocator type - using allocator_type = AllocatorType; - /// the type of an element pointer - using pointer = typename std::allocator_traits::pointer; - /// the type of an element const pointer - using const_pointer = typename std::allocator_traits::const_pointer; +/////////////////////////// +// JSON type enumeration // +/////////////////////////// - /// an iterator for a basic_json container - using iterator = iter_impl; - /// a const iterator for a basic_json container - using const_iterator = iter_impl; - /// a reverse iterator for a basic_json container - using reverse_iterator = json_reverse_iterator; - /// a const reverse iterator for a basic_json container - using const_reverse_iterator = json_reverse_iterator; +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type - /// @} +@since version 1.0.0 +*/ +enum class value_t : uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function +}; +/*! +@brief comparison operator for JSON types - /*! - @brief returns the allocator associated with the container - */ - static allocator_type get_allocator() +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string +- furthermore, each type is not smaller than itself + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) { - return allocator_type(); + return false; } + return order[static_cast(lhs)] < + order[static_cast(rhs)]; +} - /////////////////////////// - // JSON value data types // - /////////////////////////// - /// @name JSON value data types - /// The data types to store a JSON value. These types are derived from - /// the template arguments passed to class @ref basic_json. - /// @{ +///////////// +// helpers // +///////////// - /*! - @brief a type for an object +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: - > An object is an unordered collection of zero or more name/value pairs, - > where a name is a string and a value is a string, number, boolean, null, - > object, or array. +template +using uncvref_t = typename std::remove_cv::type>::type; - To store objects in C++, a type is defined by the template parameters - described below. +/* +Implementation of two C++17 constructs: conjunction, negation. This is needed +to avoid evaluating all the traits in a condition - @tparam ObjectType the container to store objects (e.g., `std::map` or - `std::unordered_map`) - @tparam StringType the type of the keys or names (e.g., `std::string`). - The comparison function `std::less` is used to order elements - inside the container. - @tparam AllocatorType the allocator to use for objects (e.g., - `std::allocator`) +For example: not std::is_same::value and has_value_type::value +will not compile when T = void (on MSVC at least). Whereas +conjunction>, has_value_type>::value will +stop evaluating if negation<...>::value == false - #### Default type +Please note that those constructs must be used with caution, since symbols can +become very long quickly (which can slow down compilation and cause MSVC +internal compiler errors). Only use it when you have to (see example ahead). +*/ +template struct conjunction : std::true_type {}; +template struct conjunction : B1 {}; +template +struct conjunction : std::conditional, B1>::type {}; - With the default values for @a ObjectType (`std::map`), @a StringType - (`std::string`), and @a AllocatorType (`std::allocator`), the default - value for @a object_t is: +template struct negation : std::integral_constant < bool, !B::value > {}; - @code {.cpp} - std::map< - std::string, // key_type - basic_json, // value_type - std::less, // key_compare - std::allocator> // allocator_type - > - @endcode +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; - #### Behavior - The choice of @a object_t influences the behavior of the JSON class. With - the default type, objects have the following behavior: +////////////////// +// constructors // +////////////////// - - When all names are unique, objects will be interoperable in the sense - that all software implementations receiving that object will agree on - the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. - - Internally, name/value pairs are stored in lexicographical order of the - names. Objects will also be serialized (see @ref dump) in this order. - For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored - and serialized as `{"a": 2, "b": 1}`. - - When comparing objects, the order of the name/value pairs is irrelevant. - This makes objects interoperable in the sense that they will not be - affected by these differences. For instance, `{"b": 1, "a": 2}` and - `{"a": 2, "b": 1}` will be treated as equal. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. - - In this class, the object's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON object. +template struct external_constructor; - #### Storage - - Objects are stored as pointers in a @ref basic_json type. That is, for any - access to object values, a pointer of type `object_t*` must be - dereferenced. +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; - @sa @ref array_t -- type for an array value +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } +}; - @since version 1.0.0 +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; - @note The order name/value pairs are added to the object is *not* - preserved by the library. Therefore, iterating an object may return - name/value pairs in a different order than they were originally stored. In - fact, keys will be traversed in alphabetical order as `std::map` with - `std::less` is used by default. Please note this behavior conforms to [RFC - 7159](http://rfc7159.net/rfc7159), because any order implements the - specified "unordered" nature of JSON objects. - */ - using object_t = ObjectType, - AllocatorType>>; +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; - /*! - @brief a type for an array +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: - > An array is an ordered sequence of zero or more values. +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_type = value_t::array; + j.m_value = arr; + j.assert_invariant(); + } - To store objects in C++, a type is defined by the template parameters - explained below. + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.assert_invariant(); + } - @tparam ArrayType container type to store arrays (e.g., `std::vector` or - `std::list`) - @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + template + static void construct(BasicJsonType& j, const std::vector& arr) + { + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (bool x : arr) + { + j.m_value.array->push_back(x); + } + j.assert_invariant(); + } +}; - #### Default type +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_type = value_t::object; + j.m_value = obj; + j.assert_invariant(); + } - With the default values for @a ArrayType (`std::vector`) and @a - AllocatorType (`std::allocator`), the default value for @a array_t is: + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; - @code {.cpp} - std::vector< - basic_json, // value_type - std::allocator // allocator_type - > - @endcode + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.assert_invariant(); + } +}; - #### Limits - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the maximum depth of nesting. +//////////////////////// +// has_/is_ functions // +//////////////////////// - In this class, the array's limit of nesting is not constraint explicitly. - However, a maximum depth of nesting may be introduced by the compiler or - runtime environment. A theoretical limit can be queried by calling the - @ref max_size function of a JSON array. +/*! +@brief Helper to determine whether there's a key_type for T. - #### Storage +This helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. - Arrays are stored as pointers in a @ref basic_json type. That is, for any - access to array values, a pointer of type `array_t*` must be dereferenced. +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0, overworked in version 2.0.6 +*/ +#define NLOHMANN_JSON_HAS_HELPER(type) \ + template struct has_##type { \ + private: \ + template \ + static int detect(U &&); \ + static void detect(...); \ + public: \ + static constexpr bool value = \ + std::is_integral()))>::value; \ + } - @sa @ref object_t -- type for an object value +NLOHMANN_JSON_HAS_HELPER(mapped_type); +NLOHMANN_JSON_HAS_HELPER(key_type); +NLOHMANN_JSON_HAS_HELPER(value_type); +NLOHMANN_JSON_HAS_HELPER(iterator); - @since version 1.0.0 - */ - using array_t = ArrayType>; +#undef NLOHMANN_JSON_HAS_HELPER - /*! - @brief a type for a string - [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: - > A string is a sequence of zero or more Unicode characters. +template +struct is_compatible_object_type_impl : std::false_type {}; - To store objects in C++, a type is defined by the template parameter - described below. Unicode values are split by the JSON class into - byte-sized characters during deserialization. +template +struct is_compatible_object_type_impl +{ + static constexpr auto value = + std::is_constructible::value and + std::is_constructible::value; +}; - @tparam StringType the container to store strings (e.g., `std::string`). - Note this container is used for keys/names in objects, see @ref object_t. +template +struct is_compatible_object_type +{ + static auto constexpr value = is_compatible_object_type_impl < + conjunction>, + has_mapped_type, + has_key_type>::value, + typename BasicJsonType::object_t, CompatibleObjectType >::value; +}; - #### Default type +template +struct is_basic_json_nested_type +{ + static auto constexpr value = std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value; +}; - With the default values for @a StringType (`std::string`), the default - value for @a string_t is: +template +struct is_compatible_array_type +{ + static auto constexpr value = + conjunction>, + negation>, + negation>, + negation>, + has_value_type, + has_iterator>::value; +}; - @code {.cpp} - std::string - @endcode +template +struct is_compatible_integer_type_impl : std::false_type {}; - #### String comparison +template +struct is_compatible_integer_type_impl +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + std::is_constructible::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; - [RFC 7159](http://rfc7159.net/rfc7159) states: - > Software implementations are typically required to test names of object - > members for equality. Implementations that transform the textual - > representation into sequences of Unicode code units and then perform the - > comparison numerically, code unit by code unit, are interoperable in the - > sense that implementations will agree in all cases on equality or - > inequality of two strings. For example, implementations that compare - > strings with escaped characters unconverted may incorrectly find that - > `"a\\b"` and `"a\u005Cb"` are not equal. +template +struct is_compatible_integer_type +{ + static constexpr auto value = + is_compatible_integer_type_impl < + std::is_integral::value and + not std::is_same::value, + RealIntegerType, CompatibleNumberIntegerType > ::value; +}; - This implementation is interoperable as it does compare strings code unit - by code unit. - #### Storage +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json +{ + private: + // also check the return type of from_json + template::from_json( + std::declval(), std::declval()))>::value>> + static int detect(U&&); + static void detect(...); - String values are stored as pointers in a @ref basic_json type. That is, - for any access to string values, a pointer of type `string_t*` must be - dereferenced. + public: + static constexpr bool value = std::is_integral>()))>::value; +}; - @since version 1.0.0 - */ - using string_t = StringType; +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json +{ + private: + template < + typename U, + typename = enable_if_t::from_json(std::declval()))>::value >> + static int detect(U&&); + static void detect(...); - /*! - @brief a type for a boolean + public: + static constexpr bool value = std::is_integral>()))>::value; +}; - [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a - type which differentiates the two literals `true` and `false`. +// This trait checks if BasicJsonType::json_serializer::to_json exists +template +struct has_to_json +{ + private: + template::to_json( + std::declval(), std::declval()))> + static int detect(U&&); + static void detect(...); - To store objects in C++, a type is defined by the template parameter @a - BooleanType which chooses the type to use. + public: + static constexpr bool value = std::is_integral>()))>::value; +}; - #### Default type - With the default values for @a BooleanType (`bool`), the default value for - @a boolean_t is: +///////////// +// to_json // +///////////// - @code {.cpp} - bool - @endcode +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); +} - #### Storage +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} - Boolean values are stored directly inside a @ref basic_json type. +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} - @since version 1.0.0 - */ - using boolean_t = BooleanType; +template < + typename BasicJsonType, typename CompatibleNumberUnsignedType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} - /*! - @brief a type for a number (integer) +template < + typename BasicJsonType, typename CompatibleNumberIntegerType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. +template::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type::type; + external_constructor::construct(j, static_cast(e)); +} - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. +template +void to_json(BasicJsonType& j, const std::vector& e) +{ + external_constructor::construct(j, e); +} - To store integer numbers in C++, a type is defined by the template - parameter @a NumberIntegerType which chooses the type to use. +template < + typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < + is_compatible_array_type::value or + std::is_same::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} - #### Default type +template < + typename BasicJsonType, typename CompatibleObjectType, + enable_if_t::value, + int> = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& arr) +{ + external_constructor::construct(j, arr); +} - With the default values for @a NumberIntegerType (`int64_t`), the default - value for @a number_integer_t is: +template ::value, + int> = 0> +void to_json(BasicJsonType& j, T (&arr)[N]) +{ + external_constructor::construct(j, arr); +} - @code {.cpp} - int64_t - @endcode +/////////////// +// from_json // +/////////////// + +// overloads for basic_json template parameters +template::value and + not std::is_same::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast( + *j.template get_ptr()); + break; + } + default: + { + JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name())); + } + } +} - #### Default behavior +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (not j.is_boolean()) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + j.type_name())); + } + b = *j.template get_ptr(); +} - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (not j.is_string()) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + j.type_name())); + } + s = *j.template get_ptr(); +} - #### Limits +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} - When the default type is used, the maximal integer number that can be - stored is `9223372036854775807` (INT64_MAX) and the minimal integer number - that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers - that are out of range will yield over/underflow when used in a - constructor. During deserialization, too large or small integer numbers - will be automatically be stored as @ref number_unsigned_t or @ref - number_float_t. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. +template::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} - As this range is a subrange of the exactly supported range [INT64_MIN, - INT64_MAX], this class's integer type is interoperable. +template +void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } + arr = *j.template get_ptr(); +} - #### Storage +// forward_list doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } - Integer number values are stored directly inside a @ref basic_json type. + for (auto it = j.rbegin(), end = j.rend(); it != end; ++it) + { + l.push_front(it->template get()); + } +} - @sa @ref number_float_t -- type for number values (floating-point) +template +void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0>) +{ + using std::begin; + using std::end; - @sa @ref number_unsigned_t -- type for number values (unsigned integer) + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} - @since version 1.0.0 - */ - using number_integer_t = NumberIntegerType; +template +auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1>) +-> decltype( + arr.reserve(std::declval()), + void()) +{ + using std::begin; + using std::end; - /*! - @brief a type for a number (unsigned) + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. +template::value and + std::is_convertible::value and + not std::is_same::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleArrayType& arr) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. + from_json_array_impl(j, arr, priority_tag<1> {}); +} - To store unsigned integer numbers in C++, a type is defined by the - template parameter @a NumberUnsignedType which chooses the type to use. +template::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleObjectType& obj) +{ + if (not j.is_object()) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + j.type_name())); + } - #### Default type + auto inner_object = j.template get_ptr(); + using std::begin; + using std::end; + // we could avoid the assignment, but this might require a for loop, which + // might be less efficient than the container constructor for some + // containers (would it?) + obj = CompatibleObjectType(begin(*inner_object), end(*inner_object)); +} - With the default values for @a NumberUnsignedType (`uint64_t`), the - default value for @a number_unsigned_t is: +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + default: + { + JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name())); + } + } +} - @code {.cpp} - uint64_t - @endcode +struct to_json_fn +{ + private: + template + auto call(BasicJsonType& j, T&& val, priority_tag<1>) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } - #### Default behavior + template + void call(BasicJsonType&, T&&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find to_json() method in T's namespace"); + } - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in integer literals lead to an interpretation as octal - number. Internally, the value will be stored as decimal number. For - instance, the C++ integer literal `010` will be serialized to `8`. - During deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. + public: + template + void operator()(BasicJsonType& j, T&& val) const + noexcept(noexcept(std::declval().call(j, std::forward(val), priority_tag<1> {}))) + { + return call(j, std::forward(val), priority_tag<1> {}); + } +}; - #### Limits +struct from_json_fn +{ + private: + template + auto call(const BasicJsonType& j, T& val, priority_tag<1>) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); + } - [RFC 7159](http://rfc7159.net/rfc7159) specifies: - > An implementation may set limits on the range and precision of numbers. + template + void call(const BasicJsonType&, T&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find from_json() method in T's namespace"); + } - When the default type is used, the maximal integer number that can be - stored is `18446744073709551615` (UINT64_MAX) and the minimal integer - number that can be stored is `0`. Integer numbers that are out of range - will yield over/underflow when used in a constructor. During - deserialization, too large or small integer numbers will be automatically - be stored as @ref number_integer_t or @ref number_float_t. + public: + template + void operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(std::declval().call(j, val, priority_tag<1> {}))) + { + return call(j, val, priority_tag<1> {}); + } +}; - [RFC 7159](http://rfc7159.net/rfc7159) further states: - > Note that when such software is used, numbers that are integers and are - > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense - > that implementations will agree exactly on their numeric values. +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; - As this range is a subrange (when considered in conjunction with the - number_integer_t type) of the exactly supported range [0, UINT64_MAX], - this class's integer type is interoperable. +template +constexpr T static_const::value; +} // namespace detail - #### Storage - Integer number values are stored directly inside a @ref basic_json type. +/// namespace to hold default `to_json` / `from_json` functions +namespace +{ +constexpr const auto& to_json = detail::static_const::value; +constexpr const auto& from_json = detail::static_const::value; +} - @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref number_integer_t -- type for number values (integer) - @since version 2.0.0 - */ - using number_unsigned_t = NumberUnsignedType; +/*! +@brief default JSONSerializer template argument +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer +{ /*! - @brief a type for a number (floating-point) - - [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: - > The representation of numbers is similar to that used in most - > programming languages. A number is represented in base 10 using decimal - > digits. It contains an integer component that may be prefixed with an - > optional minus sign, which may be followed by a fraction part and/or an - > exponent part. Leading zeros are not allowed. (...) Numeric values that - > cannot be represented in the grammar below (such as Infinity and NaN) - > are not permitted. - - This description includes both integer and floating-point numbers. - However, C++ allows more precise storage if it is known whether the number - is a signed integer, an unsigned integer or a floating-point number. - Therefore, three different types, @ref number_integer_t, @ref - number_unsigned_t and @ref number_float_t are used. - - To store floating-point numbers in C++, a type is defined by the template - parameter @a NumberFloatType which chooses the type to use. - - #### Default type - - With the default values for @a NumberFloatType (`double`), the default - value for @a number_float_t is: - - @code {.cpp} - double - @endcode - - #### Default behavior - - - The restrictions about leading zeros is not enforced in C++. Instead, - leading zeros in floating-point literals will be ignored. Internally, - the value will be stored as decimal number. For instance, the C++ - floating-point literal `01.2` will be serialized to `1.2`. During - deserialization, leading zeros yield an error. - - Not-a-number (NaN) values will be serialized to `null`. - - #### Limits - - [RFC 7159](http://rfc7159.net/rfc7159) states: - > This specification allows implementations to set limits on the range and - > precision of numbers accepted. Since software that implements IEEE - > 754-2008 binary64 (double precision) numbers is generally available and - > widely used, good interoperability can be achieved by implementations - > that expect no more precision or range than these provide, in the sense - > that implementations will approximate JSON numbers within the expected - > precision. - - This implementation does exactly follow this approach, as it uses double - precision floating-point numbers. Note values smaller than - `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` - will be stored as NaN internally and be serialized to `null`. - - #### Storage - - Floating-point number values are stored directly inside a @ref basic_json - type. - - @sa @ref number_integer_t -- type for number values (integer) + @brief convert a JSON value to any value type - @sa @ref number_unsigned_t -- type for number values (unsigned integer) + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). - @since version 1.0.0 + @param[in] j JSON value to read from + @param[in,out] val value to write to */ - using number_float_t = NumberFloatType; - - /// @} - - - /////////////////////////// - // JSON type enumeration // - /////////////////////////// + template + static void from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + { + ::nlohmann::from_json(std::forward(j), val); + } /*! - @brief the JSON type enumeration + @brief convert any value type to a JSON value - This enumeration collects the different JSON types. It is internally used - to distinguish the stored values, and the functions @ref is_null(), @ref - is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref - is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and - @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and - @ref is_structured() rely on it. + This function is usually called by the constructors of the @ref basic_json + class. - @note There are three enumeration entries (number_integer, - number_unsigned, and number_float), because the library distinguishes - these three types for numbers: @ref number_unsigned_t is used for unsigned - integers, @ref number_integer_t is used for signed integers, and @ref - number_float_t is used for floating-point numbers or to approximate - integers which do not fit in the limits of their respective type. + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template + static void to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; - @sa @ref basic_json(const value_t value_type) -- create a JSON value with - the default value for a given type - @since version 1.0.0 - */ - enum class value_t : uint8_t - { - null, ///< null value - object, ///< object (unordered set of name/value pairs) - array, ///< array (ordered collection of values) - string, ///< string value - boolean, ///< boolean value - number_integer, ///< number value (signed integer) - number_unsigned, ///< number value (unsigned integer) - number_float, ///< number value (floating-point) - discarded ///< discarded by the the parser callback function - }; +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) +@tparam JSONSerializer the serializer to resolve internal calls to `to_json()` +and `from_json()` (@ref adl_serializer by default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null + value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the + class has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 +@nosubgrouping +*/ +template < + template class ObjectType = std::map, + template class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = adl_serializer + > +class basic_json +{ private: + template friend struct detail::external_constructor; + /// workaround type for MSVC + using basic_json_t = basic_json; - /// helper for exception-safe object creation - template - static T* create(Args&& ... args) + public: + using value_t = detail::value_t; + // forward declarations + template class iter_impl; + template class json_reverse_iterator; + class json_pointer; + template + using json_serializer = JSONSerializer; + + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + /// @copydoc detail::exception + using exception = detail::exception; + /// @copydoc detail::parse_error + using parse_error = detail::parse_error; + /// @copydoc detail::invalid_iterator + using invalid_iterator = detail::invalid_iterator; + /// @copydoc detail::type_error + using type_error = detail::type_error; + /// @copydoc detail::out_of_range + using out_of_range = detail::out_of_range; + /// @copydoc detail::other_error + using other_error = detail::other_error; + + /// @} + + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl; + /// a const iterator for a basic_json container + using const_iterator = iter_impl; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() { - AllocatorType alloc; - auto deleter = [&](T * object) - { - alloc.deallocate(object, 1); - }; - std::unique_ptr object(alloc.allocate(1), deleter); - alloc.construct(object.get(), std::forward(args)...); - assert(object.get() != nullptr); - return object.release(); + return allocator_type(); } - //////////////////////// - // JSON value storage // - //////////////////////// - /*! - @brief a JSON value + @brief returns version information on the library - The actual storage for a JSON value of the @ref basic_json class. This - union combines the different storage types for the JSON value types - defined in @ref value_t. + This function returns a JSON object with information about the library, + including the version number and information on the platform and compiler. - JSON type | value_t type | used type - --------- | --------------- | ------------------------ - object | object | pointer to @ref object_t - array | array | pointer to @ref array_t - string | string | pointer to @ref string_t - boolean | boolean | @ref boolean_t - number | number_integer | @ref number_integer_t - number | number_unsigned | @ref number_unsigned_t - number | number_float | @ref number_float_t - null | null | *no value is stored* + @return JSON object holding version information + key | description + ----------- | --------------- + `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). + `copyright` | The copyright line for the library as string. + `name` | The name of the library as string. + `platform` | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`. + `url` | The URL of the project as string. + `version` | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string). - @note Variable-length types (objects, arrays, and strings) are stored as - pointers. The size of the union should not exceed 64 bits if the default - value types are used. + @liveexample{The following code shows an example output of the `meta()` + function.,meta} - @since version 1.0.0 + @complexity Constant. + + @since 2.1.0 */ - union json_value + static basic_json meta() { - /// object (stored with pointer to save storage) - object_t* object; - /// array (stored with pointer to save storage) - array_t* array; - /// string (stored with pointer to save storage) - string_t* string; - /// boolean - boolean_t boolean; - /// number (integer) - number_integer_t number_integer; - /// number (unsigned integer) - number_unsigned_t number_unsigned; - /// number (floating-point) - number_float_t number_float; + basic_json result; - /// default constructor (for null values) - json_value() = default; - /// constructor for booleans - json_value(boolean_t v) noexcept : boolean(v) {} - /// constructor for numbers (integer) - json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} - /// constructor for numbers (floating-point) - json_value(number_float_t v) noexcept : number_float(v) {} - /// constructor for empty values of a given type - json_value(value_t t) + result["copyright"] = "(C) 2013-2017 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"] = { - switch (t) - { - case value_t::object: - { - object = create(); - break; - } + {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1} + }; - case value_t::array: - { - array = create(); - break; - } +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif - case value_t::string: - { - string = create(""); - break; - } +#if defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif - case value_t::boolean: - { - boolean = boolean_t(false); - break; - } +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } - case value_t::number_integer: - { - number_integer = number_integer_t(0); - break; - } - case value_t::number_unsigned: - { - number_unsigned = number_unsigned_t(0); - break; - } + /////////////////////////// + // JSON value data types // + /////////////////////////// - case value_t::number_float: - { - number_float = number_float_t(0.0); - break; - } + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ - case value_t::null: - { - break; - } + /*! + @brief a type for an object - default: - { - if (t == value_t::null) - { - throw std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.0.10"); // LCOV_EXCL_LINE - } - break; - } - } - } + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. - /// constructor for strings - json_value(const string_t& value) - { - string = create(value); - } + To store objects in C++, a type is defined by the template parameters + described below. - /// constructor for objects - json_value(const object_t& value) - { - object = create(value); - } + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) - /// constructor for arrays - json_value(const array_t& value) - { - array = create(value); - } - }; + #### Default type - /*! - @brief checks the class invariants + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: - This function asserts the class invariants. It needs to be called at the - end of every constructor to make sure that created objects respect the - invariant. Furthermore, it has to be called each time the type of a JSON - value is changed, because the invariant expresses a relationship between - @a m_type and @a m_value. - */ - void assert_invariant() const - { - assert(m_type != value_t::object or m_value.object != nullptr); - assert(m_type != value_t::array or m_value.array != nullptr); - assert(m_type != value_t::string or m_value.string != nullptr); - } + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less, // key_compare + std::allocator> // allocator_type + > + @endcode - public: - ////////////////////////// - // JSON parser callback // - ////////////////////////// + #### Behavior - /*! - @brief JSON callback events + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: - This enumeration lists the parser events that can trigger calling a - callback function of type @ref parser_callback_t during parsing. + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. - @image html callback_events.png "Example when certain parse events are triggered" + #### Limits - @since version 1.0.0 - */ - enum class parse_event_t : uint8_t - { - /// the parser read `{` and started to process a JSON object - object_start, - /// the parser read `}` and finished processing a JSON object - object_end, - /// the parser read `[` and started to process a JSON array - array_start, - /// the parser read `]` and finished processing a JSON array - array_end, - /// the parser read a key of a value in an object - key, - /// the parser finished reading a JSON value - value - }; + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType, + AllocatorType>>; /*! - @brief per-element parser callback type + @brief a type for an array - With a parser callback function, the result of parsing a JSON text can be - influenced. When passed to @ref parse(std::istream&, const - parser_callback_t) or @ref parse(const CharT, const parser_callback_t), - it is called on certain events (passed as @ref parse_event_t via parameter - @a event) with a set recursion depth @a depth and context JSON value - @a parsed. The return value of the callback function is a boolean - indicating whether the element that emitted the callback shall be kept or - not. + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. - We distinguish six scenarios (determined by the event type) in which the - callback function can be called. The following table describes the values - of the parameters @a depth, @a event, and @a parsed. + To store objects in C++, a type is defined by the template parameters + explained below. - parameter @a event | description | parameter @a depth | parameter @a parsed - ------------------ | ----------- | ------------------ | ------------------- - parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded - parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key - parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object - parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded - parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array - parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) - @image html callback_events.png "Example when certain parse events are triggered" + #### Default type - Discarding a value (i.e., returning `false`) has different effects - depending on the context in which function was called: + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: - - Discarded values in structured types are skipped. That is, the parser - will behave as if the discarded value was never read. - - In case a value outside a structured type is skipped, it is replaced - with `null`. This case happens if the top-level element is skipped. + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator // allocator_type + > + @endcode - @param[in] depth the depth of the recursion during parsing + #### Limits - @param[in] event an event of type parse_event_t indicating the context in - the callback function has been called + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. - @param[in,out] parsed the current intermediate parse result; note that - writing to this value has no effect for parse_event_t::key events + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. - @return Whether the JSON value which called the function during parsing - should be kept (`true`) or not (`false`). In the latter case, it is either - skipped completely or replaced by an empty discarded object. + #### Storage - @sa @ref parse(std::istream&, parser_callback_t) or - @ref parse(const CharT, const parser_callback_t) for examples + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value @since version 1.0.0 */ - using parser_callback_t = std::function; + using array_t = ArrayType>; + /*! + @brief a type for a string - ////////////////// - // constructors // - ////////////////// + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. - /// @name constructors and destructors - /// Constructors of class @ref basic_json, copy/move constructor, copy - /// assignment, static functions creating objects, and the destructor. - /// @{ + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. - /*! - @brief create an empty value with a given type + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. - Create an empty JSON value with a given type. The value will be default - initialized with an empty value which depends on the type: + #### Default type - Value type | initial value - ----------- | ------------- - null | `null` - boolean | `false` - string | `""` - number | `0` - object | `{}` - array | `[]` + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: - @param[in] value_type the type of the value to create + @code {.cpp} + std::string + @endcode - @complexity Constant. + #### Encoding - @throw std::bad_alloc if allocation for object, array, or string value - fails + Strings are stored in UTF-8 encoding. Therefore, functions like + `std::string::size()` or `std::string::length()` return the number of + bytes in the string rather than the number of characters or glyphs. - @liveexample{The following code shows the constructor for different @ref - value_t values,basic_json__value_t} + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - @sa @ref basic_json(boolean_t value) -- create a boolean value - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const object_t&) -- create a object value - @sa @ref basic_json(const array_t&) -- create a array value - @sa @ref basic_json(const number_float_t) -- create a number - (floating-point) value - @sa @ref basic_json(const number_integer_t) -- create a number (integer) - value - @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) - value + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. @since version 1.0.0 */ - basic_json(const value_t value_type) - : m_type(value_type), m_value(value_type) - { - assert_invariant(); - } + using string_t = StringType; /*! - @brief create a null object + @brief a type for a boolean - Create a `null` JSON value. It either takes a null pointer as parameter - (explicitly creating `null`) or no parameter (implicitly creating `null`). - The passed null pointer itself is not read -- it is only used to choose - the right constructor. + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. - @complexity Constant. + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. - @exceptionsafety No-throw guarantee: this constructor never throws - exceptions. + #### Default type - @liveexample{The following code shows the constructor with and without a - null pointer parameter.,basic_json__nullptr_t} + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. @since version 1.0.0 */ - basic_json(std::nullptr_t = nullptr) noexcept - : basic_json(value_t::null) - { - assert_invariant(); - } + using boolean_t = BooleanType; /*! - @brief create an object (explicit) + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. - Create an object JSON value with a given content. + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. - @param[in] val a value for the object + #### Default type - @complexity Linear in the size of the passed @a val. + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: - @throw std::bad_alloc if allocation for object value fails + @code {.cpp} + int64_t + @endcode - @liveexample{The following code shows the constructor with an @ref - object_t parameter.,basic_json__object_t} + #### Default behavior - @sa @ref basic_json(const CompatibleObjectType&) -- create an object value - from a compatible STL container + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. - @since version 1.0.0 - */ - basic_json(const object_t& val) - : m_type(value_t::object), m_value(val) - { - assert_invariant(); - } + #### Limits - /*! - @brief create an object (implicit) + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. - Create an object JSON value with a given content. This constructor allows - any type @a CompatibleObjectType that can be used to construct values of - type @ref object_t. + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. - @tparam CompatibleObjectType An object type whose `key_type` and - `value_type` is compatible to @ref object_t. Examples include `std::map`, - `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with - a `key_type` of `std::string`, and a `value_type` from which a @ref - basic_json value can be constructed. + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. - @param[in] val a value for the object + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. - @complexity Linear in the size of the passed @a val. + #### Storage - @throw std::bad_alloc if allocation for object value fails + Integer number values are stored directly inside a @ref basic_json type. - @liveexample{The following code shows the constructor with several - compatible object type parameters.,basic_json__CompatibleObjectType} + @sa @ref number_float_t -- type for number values (floating-point) - @sa @ref basic_json(const object_t&) -- create an object value + @sa @ref number_unsigned_t -- type for number values (unsigned integer) @since version 1.0.0 */ - template::value and - std::is_constructible::value, int>::type = 0> - basic_json(const CompatibleObjectType& val) - : m_type(value_t::object) - { - using std::begin; - using std::end; - m_value.object = create(begin(val), end(val)); - assert_invariant(); - } + using number_integer_t = NumberIntegerType; /*! - @brief create an array (explicit) + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. - Create an array JSON value with a given content. + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. - @param[in] val a value for the array + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. - @complexity Linear in the size of the passed @a val. + #### Default type - @throw std::bad_alloc if allocation for array value fails + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: - @liveexample{The following code shows the constructor with an @ref array_t - parameter.,basic_json__array_t} + @code {.cpp} + uint64_t + @endcode - @sa @ref basic_json(const CompatibleArrayType&) -- create an array value - from a compatible STL containers + #### Default behavior - @since version 1.0.0 - */ - basic_json(const array_t& val) - : m_type(value_t::array), m_value(val) - { - assert_invariant(); - } + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. - /*! - @brief create an array (implicit) + #### Limits - Create an array JSON value with a given content. This constructor allows - any type @a CompatibleArrayType that can be used to construct values of - type @ref array_t. + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. - @tparam CompatibleArrayType An object type whose `value_type` is - compatible to @ref array_t. Examples include `std::vector`, `std::deque`, - `std::list`, `std::forward_list`, `std::array`, `std::set`, - `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a - `value_type` from which a @ref basic_json value can be constructed. + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. - @param[in] val a value for the array + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. - @complexity Linear in the size of the passed @a val. + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. - @throw std::bad_alloc if allocation for array value fails + #### Storage - @liveexample{The following code shows the constructor with several - compatible array type parameters.,basic_json__CompatibleArrayType} + Integer number values are stored directly inside a @ref basic_json type. - @sa @ref basic_json(const array_t&) -- create an array value + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) - @since version 1.0.0 + @since version 2.0.0 */ - template::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type = 0> - basic_json(const CompatibleArrayType& val) - : m_type(value_t::array) - { - using std::begin; - using std::end; - m_value.array = create(begin(val), end(val)); - assert_invariant(); - } + using number_unsigned_t = NumberUnsignedType; /*! - @brief create a string (explicit) + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. - Create an string JSON value with a given content. + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. - @param[in] val a value for the string + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. - @complexity Linear in the size of the passed @a val. + #### Default type - @throw std::bad_alloc if allocation for string value fails + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: - @liveexample{The following code shows the constructor with an @ref - string_t parameter.,basic_json__string_t} + @code {.cpp} + double + @endcode - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container + #### Default behavior - @since version 1.0.0 - */ - basic_json(const string_t& val) - : m_type(value_t::string), m_value(val) - { - assert_invariant(); - } + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. - /*! - @brief create a string (explicit) + #### Limits - Create a string JSON value with a given content. + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. - @param[in] val a literal value for the string + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. - @complexity Linear in the size of the passed @a val. + #### Storage - @throw std::bad_alloc if allocation for string value fails + Floating-point number values are stored directly inside a @ref basic_json + type. - @liveexample{The following code shows the constructor with string literal - parameter.,basic_json__string_t_value_type} + @sa @ref number_integer_t -- type for number values (integer) - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container + @sa @ref number_unsigned_t -- type for number values (unsigned integer) @since version 1.0.0 */ - basic_json(const typename string_t::value_type* val) - : basic_json(string_t(val)) - { - assert_invariant(); - } + using number_float_t = NumberFloatType; - /*! - @brief create a string (implicit) + /// @} - Create a string JSON value with a given content. + private: - @param[in] val a value for the string + /// helper for exception-safe object creation + template + static T* create(Args&& ... args) + { + AllocatorType alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward(args)...); + assert(object != nullptr); + return object.release(); + } - @tparam CompatibleStringType an string type which is compatible to @ref - string_t, for instance `std::string`. + //////////////////////// + // JSON value storage // + //////////////////////// - @complexity Linear in the size of the passed @a val. + /*! + @brief a JSON value - @throw std::bad_alloc if allocation for string value fails + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. - @liveexample{The following code shows the construction of a string value - from a compatible type.,basic_json__CompatibleStringType} + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. @since version 1.0.0 */ - template::value, int>::type = 0> - basic_json(const CompatibleStringType& val) - : basic_json(string_t(val)) + union json_value { - assert_invariant(); - } + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; - /*! - @brief create a boolean (explicit) + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } - Creates a JSON boolean type from a given value. + case value_t::array: + { + array = create(); + break; + } - @param[in] val a boolean value to store + case value_t::string: + { + string = create(""); + break; + } - @complexity Constant. + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } - @liveexample{The example below demonstrates boolean - values.,basic_json__boolean_t} + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } - @since version 1.0.0 - */ - basic_json(boolean_t val) noexcept - : m_type(value_t::boolean), m_value(val) - { - assert_invariant(); - } + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } - /*! - @brief create an integer number (explicit) + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } - Create an integer number JSON value with a given content. + case value_t::null: + { + break; + } - @tparam T A helper type to remove this function via SFINAE in case @ref - number_integer_t is the same as `int`. In this case, this constructor - would have the same signature as @ref basic_json(const int value). Note - the helper type @a T is not visible in this constructor's interface. + default: + { + if (t == value_t::null) + { + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE + } + break; + } + } + } - @param[in] val an integer to create a JSON number from + /// constructor for strings + json_value(const string_t& value) + { + string = create(value); + } - @complexity Constant. + /// constructor for objects + json_value(const object_t& value) + { + object = create(value); + } - @liveexample{The example below shows the construction of an integer - number value.,basic_json__number_integer_t} + /// constructor for arrays + json_value(const array_t& value) + { + array = create(value); + } + }; - @sa @ref basic_json(const int) -- create a number value (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type + /*! + @brief checks the class invariants - @since version 1.0.0 + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. */ - template::value) and - std::is_same::value, int>::type = 0> - basic_json(const number_integer_t val) noexcept - : m_type(value_t::number_integer), m_value(val) + void assert_invariant() const { - assert_invariant(); + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); } - /*! - @brief create an integer number from an enum type (explicit) - - Create an integer number JSON value with a given content. - - @param[in] val an integer to create a JSON number from - - @note This constructor allows to pass enums directly to a constructor. As - C++ has no way of specifying the type of an anonymous enum explicitly, we - can only rely on the fact that such values implicitly convert to int. As - int may already be the same type of number_integer_t, we may need to - switch off the constructor @ref basic_json(const number_integer_t). + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// - @complexity Constant. + /*! + @brief JSON callback events - @liveexample{The example below shows the construction of an integer - number value from an anonymous enum.,basic_json__const_int} + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type + @image html callback_events.png "Example when certain parse events are triggered" @since version 1.0.0 */ - basic_json(const int val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) + enum class parse_event_t : uint8_t { - assert_invariant(); - } + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; /*! - @brief create an integer number (implicit) + @brief per-element parser callback type - Create an integer number JSON value with a given content. This constructor - allows any type @a CompatibleNumberIntegerType that can be used to - construct values of type @ref number_integer_t. + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const CharT, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. - @tparam CompatibleNumberIntegerType An integer type which is compatible to - @ref number_integer_t. Examples include the types `int`, `int32_t`, - `long`, and `short`. + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. - @param[in] val an integer to create a JSON number from + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value - @complexity Constant. + @image html callback_events.png "Example when certain parse events are triggered" - @liveexample{The example below shows the construction of several integer - number values from compatible - types.,basic_json__CompatibleIntegerNumberType} + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const int) -- create a number value (integer) + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. - @since version 1.0.0 - */ - template::value and - std::numeric_limits::is_integer and - std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type = 0> - basic_json(const CompatibleNumberIntegerType val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } + @param[in] depth the depth of the recursion during parsing - /*! - @brief create an unsigned integer number (explicit) + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called - Create an unsigned integer number JSON value with a given content. + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events - @tparam T helper type to compare number_unsigned_t and unsigned int (not - visible in) the interface. + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. - @param[in] val an integer to create a JSON number from + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const CharT, const parser_callback_t) for examples - @complexity Constant. + @since version 1.0.0 + */ + using parser_callback_t = std::function; - @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number - value (unsigned integer) from a compatible number type - @since version 2.0.0 - */ - template::value) and - std::is_same::value, int>::type = 0> - basic_json(const number_unsigned_t val) noexcept - : m_type(value_t::number_unsigned), m_value(val) - { - assert_invariant(); - } + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ /*! - @brief create an unsigned number (implicit) + @brief create an empty value with a given type - Create an unsigned number JSON value with a given content. This - constructor allows any type @a CompatibleNumberUnsignedType that can be - used to construct values of type @ref number_unsigned_t. + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: - @tparam CompatibleNumberUnsignedType An integer type which is compatible - to @ref number_unsigned_t. Examples may include the types `unsigned int`, - `uint32_t`, or `unsigned short`. + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` - @param[in] val an unsigned integer to create a JSON number from + @param[in] value_type the type of the value to create @complexity Constant. - @sa @ref basic_json(const number_unsigned_t) -- create a number value - (unsigned) + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} - @since version 2.0.0 + @since version 1.0.0 */ - template::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type = 0> - basic_json(const CompatibleNumberUnsignedType val) noexcept - : m_type(value_t::number_unsigned), - m_value(static_cast(val)) + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) { assert_invariant(); } /*! - @brief create a floating-point number (explicit) - - Create a floating-point number JSON value with a given content. - - @param[in] val a floating-point value to create a JSON number from + @brief create a null object - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is created - instead. + Create a `null` JSON value. It either takes a null pointer as parameter + (explicitly creating `null`) or no parameter (implicitly creating `null`). + The passed null pointer itself is not read -- it is only used to choose + the right constructor. @complexity Constant. - @liveexample{The following example creates several floating-point - values.,basic_json__number_float_t} + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. - @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number - value (floating-point) from a compatible number type + @liveexample{The following code shows the constructor with and without a + null pointer parameter.,basic_json__nullptr_t} @since version 1.0.0 */ - basic_json(const number_float_t val) noexcept - : m_type(value_t::number_float), m_value(val) + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) { - // replace infinity and NAN by null - if (not std::isfinite(val)) - { - m_type = value_t::null; - m_value = json_value(); - } - assert_invariant(); } /*! - @brief create an floating-point number (implicit) + @brief create a JSON value - Create an floating-point number JSON value with a given content. This - constructor allows any type @a CompatibleNumberFloatType that can be used - to construct values of type @ref number_float_t. + This is a "catch all" constructor for all compatible JSON types; that is, + types for which a `to_json()` method exsits. The constructor forwards the + parameter @a val to that method (to `json_serializer::to_json` method + with `U = uncvref_t`, to be exact). - @tparam CompatibleNumberFloatType A floating-point type which is - compatible to @ref number_float_t. Examples may include the types `float` - or `double`. + Template type @a CompatibleType includes, but is not limited to, the + following types: + - **arrays**: @ref array_t and all kinds of compatible containers such as + `std::vector`, `std::deque`, `std::list`, `std::forward_list`, + `std::array`, `std::set`, `std::unordered_set`, `std::multiset`, and + `unordered_multiset` with a `value_type` from which a @ref basic_json + value can be constructed. + - **objects**: @ref object_t and all kinds of compatible associative + containers such as `std::map`, `std::unordered_map`, `std::multimap`, + and `std::unordered_multimap` with a `key_type` compatible to + @ref string_t and a `value_type` from which a @ref basic_json value can + be constructed. + - **strings**: @ref string_t, string literals, and all compatible string + containers can be used. + - **numbers**: @ref number_integer_t, @ref number_unsigned_t, + @ref number_float_t, and all convertible number types such as `int`, + `size_t`, `int64_t`, `float` or `double` can be used. + - **boolean**: @ref boolean_t / `bool` can be used. - @param[in] val a floating-point to create a JSON number from + See the examples below. - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is - created instead. + @tparam CompatibleType a type such that: + - @a CompatibleType is not derived from `std::istream`, + - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move + constructors), + - @a CompatibleType is not a @ref basic_json nested type (e.g., + @ref json_pointer, @ref iterator, etc ...) + - @ref @ref json_serializer has a + `to_json(basic_json_t&, CompatibleType&&)` method - @complexity Constant. + @tparam U = `uncvref_t` - @liveexample{The example below shows the construction of several - floating-point number values from compatible - types.,basic_json__CompatibleNumberFloatType} + @param[in] val the value to be forwarded - @sa @ref basic_json(const number_float_t) -- create a number value - (floating-point) + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. - @since version 1.0.0 + @throw what `json_serializer::to_json()` throws + + @liveexample{The following code shows the constructor with several + compatible types.,basic_json__CompatibleType} + + @since version 2.1.0 */ - template::value and - std::is_floating_point::value>::type> - basic_json(const CompatibleNumberFloatType val) noexcept - : basic_json(number_float_t(val)) + template, + detail::enable_if_t::value and + not std::is_same::value and + not detail::is_basic_json_nested_type< + basic_json_t, U>::value and + detail::has_to_json::value, + int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer::to_json( + std::declval(), std::forward(val)))) { + JSONSerializer::to_json(*this, std::forward(val)); assert_invariant(); } @@ -1613,10 +2330,12 @@ class basic_json value_t::array and @ref value_t::object are valid); when @a type_deduction is set to `true`, this parameter has no effect - @throw std::domain_error if @a type_deduction is `false`, @a manual_type - is `value_t::object`, but @a init contains an element which is not a pair - whose first element is a string; example: `"cannot create object from - initializer list"` + @throw type_error.301 if @a type_deduction is `false`, @a manual_type is + `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string. In this case, the constructor could not + create an object. If @a type_deduction would have be `true`, an array + would have been created. See @ref object(std::initializer_list) + for an example. @complexity Linear in the size of the initializer list @a init. @@ -1654,7 +2373,7 @@ class basic_json // if object is wanted but impossible, throw an exception if (manual_type == value_t::object and not is_an_object) { - throw std::domain_error("cannot create object from initializer list"); + JSON_THROW(type_error::create(301, "cannot create object from initializer list")); } } @@ -1730,16 +2449,17 @@ class basic_json related function @ref array(std::initializer_list), there are no cases which can only be expressed by this function. That is, any initializer list @a init can also be passed to the initializer list - constructor @ref basic_json(std::initializer_list, bool, - value_t). + constructor @ref basic_json(std::initializer_list, bool, value_t). @param[in] init initializer list to create an object from (optional) @return JSON object value - @throw std::domain_error if @a init is not a pair whose first elements are - strings; thrown by - @ref basic_json(std::initializer_list, bool, value_t) + @throw type_error.301 if @a init is not a list of pairs whose first + elements are strings. In this case, no object can be created. When such a + value is passed to @ref basic_json(std::initializer_list, bool, value_t), + an array would have been created from the passed initializer list @a init. + See example below. @complexity Linear in the size of @a init. @@ -1791,10 +2511,10 @@ class basic_json The semantics depends on the different types a JSON value can have: - In case of primitive types (number, boolean, or string), @a first must be `begin()` and @a last must be `end()`. In this case, the value is - copied. Otherwise, std::out_of_range is thrown. + copied. Otherwise, invalid_iterator.204 is thrown. - In case of structured types (array, object), the constructor behaves as similar versions for `std::vector`. - - In case of a null type, std::domain_error is thrown. + - In case of a null type, invalid_iterator.206 is thrown. @tparam InputIT an input iterator type (@ref iterator or @ref const_iterator) @@ -1805,14 +2525,19 @@ class basic_json @pre Iterators @a first and @a last must be initialized. **This precondition is enforced with an assertion.** - @throw std::domain_error if iterators are not compatible; that is, do not - belong to the same JSON value; example: `"iterators are not compatible"` - @throw std::out_of_range if iterators are for a primitive type (number, - boolean, or string) where an out of range error can be detected easily; - example: `"iterators out of range"` - @throw std::bad_alloc if allocation for object, array, or string fails - @throw std::domain_error if called with a null value; example: `"cannot - use construct with iterators from null"` + @pre Range `[first, last)` is valid. Usually, this precondition cannot be + checked efficiently. Only certain edge cases are detected; see the + description of the exceptions below. + + @throw invalid_iterator.201 if iterators @a first and @a last are not + compatible (i.e., do not belong to the same JSON value). In this case, + the range `[first, last)` is undefined. + @throw invalid_iterator.204 if iterators @a first and @a last belong to a + primitive type (number, boolean, or string), but @a first does not point + to the first element any more. In this case, the range `[first, last)` is + undefined. See example code below. + @throw invalid_iterator.206 if iterators @a first and @a last belong to a + null value. In this case, the range `[first, last)` is undefined. @complexity Linear in distance between @a first and @a last. @@ -1832,7 +2557,7 @@ class basic_json // make sure iterator fits the current value if (first.m_object != last.m_object) { - throw std::domain_error("iterators are not compatible"); + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible")); } // copy type from first iterator @@ -1849,7 +2574,7 @@ class basic_json { if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) { - throw std::out_of_range("iterators out of range"); + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); } break; } @@ -1894,59 +2619,28 @@ class basic_json case value_t::object: { - m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); break; } case value_t::array: { - m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); break; } default: { - throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + + first.m_object->type_name())); } } assert_invariant(); } - /*! - @brief construct a JSON value given an input stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @deprecated This constructor is deprecated and will be removed in version - 3.0.0 to unify the interface of the library. Deserialization will be - done by stream operators or by calling one of the `parse` functions, - e.g. @ref parse(std::istream&, const parser_callback_t). That is, calls - like `json j(i);` for an input stream @a i need to be replaced by - `json j = json::parse(i);`. See the example below. - - @liveexample{The example below demonstrates constructing a JSON value from - a `std::stringstream` with and without callback - function.,basic_json__istream} - - @since version 2.0.0, deprecated in version 2.0.3, to be removed in - version 3.0.0 - */ - JSON_DEPRECATED - explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) - { - *this = parser(i, cb).parse(); - assert_invariant(); - } /////////////////////////////////////// // other constructors and destructor // @@ -1967,8 +2661,6 @@ class basic_json - The complexity is linear. - As postcondition, it holds: `other == basic_json(other)`. - @throw std::bad_alloc if allocation for object, array, or string fails. - @liveexample{The following code shows an example for the copy constructor.,basic_json__basic_json} @@ -2196,22 +2888,15 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; - // fix locale problems - ss.imbue(std::locale::classic()); - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - ss.precision(std::numeric_limits::digits10); + serializer s(ss); if (indent >= 0) { - dump(ss, true, static_cast(indent)); + s.dump(*this, true, static_cast(indent)); } else { - dump(ss, false, 0); + s.dump(*this, false, 0); } return ss.str(); @@ -2577,246 +3262,99 @@ class basic_json private: ////////////////// // value access // - ////////////////// - - /// get an object (explicit) - template::value and - std::is_convertible::value, int>::type = 0> - T get_impl(T*) const - { - if (is_object()) - { - return T(m_value.object->begin(), m_value.object->end()); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an object (explicit) - object_t get_impl(object_t*) const - { - if (is_object()) - { - return *(m_value.object); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value, int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - T to_vector; - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not std::is_same::value, int>::type = 0> - std::vector get_impl(std::vector*) const - { - if (is_array()) - { - std::vector to_vector; - to_vector.reserve(m_value.array->size()); - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not has_mapped_type::value, int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - return T(m_value.array->begin(), m_value.array->end()); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - array_t get_impl(array_t*) const - { - if (is_array()) - { - return *(m_value.array); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get a string (explicit) - template::value, int>::type = 0> - T get_impl(T*) const - { - if (is_string()) - { - return *m_value.string; - } - else - { - throw std::domain_error("type must be string, but is " + type_name()); - } - } - - /// get a number (explicit) - template::value, int>::type = 0> - T get_impl(T*) const - { - switch (m_type) - { - case value_t::number_integer: - { - return static_cast(m_value.number_integer); - } - - case value_t::number_unsigned: - { - return static_cast(m_value.number_unsigned); - } - - case value_t::number_float: - { - return static_cast(m_value.number_float); - } - - default: - { - throw std::domain_error("type must be number, but is " + type_name()); - } - } - } + ////////////////// /// get a boolean (explicit) - constexpr boolean_t get_impl(boolean_t*) const + boolean_t get_impl(boolean_t* /*unused*/) const { - return is_boolean() - ? m_value.boolean - : throw std::domain_error("type must be boolean, but is " + type_name()); + if (is_boolean()) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + type_name())); } /// get a pointer to the value (object) - object_t* get_impl_ptr(object_t*) noexcept + object_t* get_impl_ptr(object_t* /*unused*/) noexcept { return is_object() ? m_value.object : nullptr; } /// get a pointer to the value (object) - constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept { return is_object() ? m_value.object : nullptr; } /// get a pointer to the value (array) - array_t* get_impl_ptr(array_t*) noexcept + array_t* get_impl_ptr(array_t* /*unused*/) noexcept { return is_array() ? m_value.array : nullptr; } /// get a pointer to the value (array) - constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept { return is_array() ? m_value.array : nullptr; } /// get a pointer to the value (string) - string_t* get_impl_ptr(string_t*) noexcept + string_t* get_impl_ptr(string_t* /*unused*/) noexcept { return is_string() ? m_value.string : nullptr; } /// get a pointer to the value (string) - constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept { return is_string() ? m_value.string : nullptr; } /// get a pointer to the value (boolean) - boolean_t* get_impl_ptr(boolean_t*) noexcept + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept { return is_boolean() ? &m_value.boolean : nullptr; } /// get a pointer to the value (boolean) - constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept { return is_boolean() ? &m_value.boolean : nullptr; } /// get a pointer to the value (integer number) - number_integer_t* get_impl_ptr(number_integer_t*) noexcept + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept { return is_number_integer() ? &m_value.number_integer : nullptr; } /// get a pointer to the value (integer number) - constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept { return is_number_integer() ? &m_value.number_integer : nullptr; } /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept { return is_number_unsigned() ? &m_value.number_unsigned : nullptr; } /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept { return is_number_unsigned() ? &m_value.number_unsigned : nullptr; } /// get a pointer to the value (floating-point number) - number_float_t* get_impl_ptr(number_float_t*) noexcept + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept { return is_number_float() ? &m_value.number_float : nullptr; } /// get a pointer to the value (floating-point number) - constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept { return is_number_float() ? &m_value.number_float : nullptr; } @@ -2829,7 +3367,7 @@ class basic_json @tparam ThisType will be deduced as `basic_json` or `const basic_json` - @throw std::domain_error if ReferenceType does not match underlying value + @throw type_error.303 if ReferenceType does not match underlying value type of the current JSON */ template @@ -2845,34 +3383,68 @@ class basic_json { return *ptr; } - else - { - throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + - obj.type_name()); - } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + obj.type_name())); } public: - /// @name value access /// Direct access to the stored value of a JSON value. /// @{ + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template < + typename BasicJsonType, + detail::enable_if_t::type, + basic_json_t>::value, + int> = 0 > + basic_json get() const + { + return *this; + } + /*! @brief get a value (explicit) - Explicit type conversion between the JSON value and a compatible value. + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode - @return copy of the JSON value, converted to type @a ValueType + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const @ref basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const @ref basic_json&)` - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON; example: `"type must be object, but is null"` + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type - @complexity Linear in the size of the JSON value. + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws @liveexample{The example below shows several conversions from JSON values to other types. There a few things to note: (1) Floating-point numbers can @@ -2881,21 +3453,75 @@ class basic_json associative containers such as `std::unordered_map`.,get__ValueType_const} - @internal - The idea of using a casted null pointer to choose the correct - implementation is from . - @endinternal + @since version 2.1.0 + */ + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t < + not std::is_same::value and + detail::has_from_json::value and + not detail::has_non_default_from_json::value, + int > = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + static_assert(std::is_default_constructible::value, + "types must be DefaultConstructible when used with get()"); - @sa @ref operator ValueType() const for implicit conversion - @sa @ref get() for pointer-member access + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + } - @since version 1.0.0 + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const @ref basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 */ - template::value, int>::type = 0> - ValueType get() const + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t::value and + detail::has_non_default_from_json::value, int> = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) { - return get_impl(static_cast(nullptr)); + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return JSONSerializer::from_json(*this); } /*! @@ -3025,7 +3651,7 @@ class basic_json /*! @brief get a reference value (implicit) - Implict reference access to the internally stored JSON value. No copies + Implicit reference access to the internally stored JSON value. No copies are made. @warning Writing data to the referee of the result yields an undefined @@ -3037,10 +3663,10 @@ class basic_json @return reference to the internally stored JSON value if the requested reference type @a ReferenceType fits to the JSON value; throws - std::domain_error otherwise + type_error.303 otherwise - @throw std::domain_error in case passed type @a ReferenceType is - incompatible with the stored JSON value + @throw type_error.303 in case passed type @a ReferenceType is incompatible + with the stored JSON value; see example below @complexity Constant. @@ -3083,8 +3709,9 @@ class basic_json @return copy of the JSON value, converted to type @a ValueType - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON, thrown by @ref get() const + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below @complexity Linear in the size of the JSON value. @@ -3100,8 +3727,11 @@ class basic_json template < typename ValueType, typename std::enable_if < not std::is_pointer::value and not std::is_same::value -#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 +#ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same>::value +#endif +#if defined(_MSC_VER) && _MSC_VER >1900 && defined(_HAS_CXX17) && _HAS_CXX17 == 1 // fix for issue #464 + and not std::is_same::value #endif , int >::type = 0 > operator ValueType() const @@ -3131,36 +3761,40 @@ class basic_json @return reference to the element at index @a idx - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. - @complexity Constant. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how array elements can be read and - written using `at()`.,at__size_type} + @complexity Constant. @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__size_type} */ reference at(size_type idx) { // at only works for arrays if (is_array()) { - try + JSON_TRY { return m_value.array->at(idx); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3174,36 +3808,40 @@ class basic_json @return const reference to the element at index @a idx - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. - @complexity Constant. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how array elements can be read using - `at()`.,at__size_type_const} + @complexity Constant. @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__size_type_const} */ const_reference at(size_type idx) const { // at only works for arrays if (is_array()) { - try + JSON_TRY { return m_value.array->at(idx); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3217,40 +3855,44 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. - @complexity Logarithmic in the size of the container. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how object elements can be read and - written using `at()`.,at__object_t_key_type} + @complexity Logarithmic in the size of the container. @sa @ref operator[](const typename object_t::key_type&) for unchecked access by reference @sa @ref value() for access by value with a default value @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__object_t_key_type} */ reference at(const typename object_t::key_type& key) { // at only works for objects if (is_object()) { - try + JSON_TRY { return m_value.object->at(key); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3264,40 +3906,44 @@ class basic_json @return const reference to the element at key @a key - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. - @complexity Logarithmic in the size of the container. + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. - @liveexample{The example below shows how object elements can be read using - `at()`.,at__object_t_key_type_const} + @complexity Logarithmic in the size of the container. @sa @ref operator[](const typename object_t::key_type&) for unchecked access by reference @sa @ref value() for access by value with a default value @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__object_t_key_type_const} */ const_reference at(const typename object_t::key_type& key) const { // at only works for objects if (is_object()) { - try + JSON_TRY { return m_value.object->at(key); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3314,8 +3960,8 @@ class basic_json @return reference to the element at index @a idx - @throw std::domain_error if JSON is not an array or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an array or null; in that + cases, using the [] operator with an index makes no sense. @complexity Constant if @a idx is in the range of the array. Otherwise linear in `idx - size()`. @@ -3349,10 +3995,8 @@ class basic_json return m_value.array->operator[](idx); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3364,8 +4008,8 @@ class basic_json @return const reference to the element at index @a idx - @throw std::domain_error if JSON is not an array; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an array; in that cases, + using the [] operator with an index makes no sense. @complexity Constant. @@ -3381,10 +4025,8 @@ class basic_json { return m_value.array->operator[](idx); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3400,8 +4042,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3429,10 +4071,8 @@ class basic_json { return m_value.object->operator[](key); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3451,8 +4091,8 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3473,10 +4113,8 @@ class basic_json assert(m_value.object->find(key) != m_value.object->end()); return m_value.object->find(key)->second; } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3492,8 +4130,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3527,8 +4165,8 @@ class basic_json @return const reference to the element at key @a key - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3560,8 +4198,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3590,10 +4228,8 @@ class basic_json { return m_value.object->operator[](key); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3612,8 +4248,8 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3635,10 +4271,8 @@ class basic_json assert(m_value.object->find(key) != m_value.object->end()); return m_value.object->find(key)->second; } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3651,7 +4285,7 @@ class basic_json @code {.cpp} try { return at(key); - } catch(std::out_of_range) { + } catch(out_of_range) { return default_value; } @endcode @@ -3674,8 +4308,8 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` + @throw type_error.306 if the JSON value is not an objec; in that cases, + using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3702,14 +4336,12 @@ class basic_json { return *it; } - else - { - return default_value; - } + + return default_value; } else { - throw std::domain_error("cannot use value() with " + type_name()); + JSON_THROW(type_error::create(306, "cannot use value() with " + type_name())); } } @@ -3732,7 +4364,7 @@ class basic_json @code {.cpp} try { return at(ptr); - } catch(std::out_of_range) { + } catch(out_of_range) { return default_value; } @endcode @@ -3751,8 +4383,8 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` + @throw type_error.306 if the JSON value is not an objec; in that cases, + using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3771,19 +4403,17 @@ class basic_json if (is_object()) { // if pointer resolves a value, return it or use default value - try + JSON_TRY { return ptr.get_checked(this); } - catch (std::out_of_range&) + JSON_CATCH (out_of_range&) { return default_value; } } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } + + JSON_THROW(type_error::create(306, "cannot use value() with " + type_name())); } /*! @@ -3812,7 +4442,7 @@ class basic_json assertions**). @post The JSON value remains unchanged. - @throw std::out_of_range when called on `null` value + @throw invalid_iterator.214 when called on `null` value @liveexample{The following code shows an example for `front()`.,front} @@ -3855,7 +4485,8 @@ class basic_json assertions**). @post The JSON value remains unchanged. - @throw std::out_of_range when called on `null` value. + @throw invalid_iterator.214 when called on a `null` value. See example + below. @liveexample{The following code shows an example for `back()`.,back} @@ -3899,17 +4530,18 @@ class basic_json @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on an iterator which does not belong to - the current JSON value; example: `"iterator does not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.202 if called on an iterator which does not belong + to the current JSON value; example: `"iterator does not fit current + value"` + @throw invalid_iterator.205 if called on a primitive type with invalid iterator (i.e., any iterator which is not `begin()`); example: `"iterator out of range"` @complexity The complexity depends on the type: - objects: amortized constant - - arrays: linear in distance between pos and the end of the container + - arrays: linear in distance between @a pos and the end of the container - strings: linear in the length of the string - other types: constant @@ -3934,7 +4566,7 @@ class basic_json // make sure iterator fits the current value if (this != pos.m_object) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } IteratorType result = end(); @@ -3949,7 +4581,7 @@ class basic_json { if (not pos.m_it.primitive_iterator.is_begin()) { - throw std::out_of_range("iterator out of range"); + JSON_THROW(invalid_iterator::create(205, "iterator out of range")); } if (is_string()) @@ -3979,7 +4611,7 @@ class basic_json default: { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4006,11 +4638,11 @@ class basic_json @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on iterators which does not belong to - the current JSON value; example: `"iterators do not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.203 if called on iterators which does not belong + to the current JSON value; example: `"iterators do not fit current value"` + @throw invalid_iterator.204 if called on a primitive type with invalid iterators (i.e., if `first != begin()` and `last != end()`); example: `"iterators out of range"` @@ -4041,7 +4673,7 @@ class basic_json // make sure iterator fits the current value if (this != first.m_object or this != last.m_object) { - throw std::domain_error("iterators do not fit current value"); + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value")); } IteratorType result = end(); @@ -4056,7 +4688,7 @@ class basic_json { if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) { - throw std::out_of_range("iterators out of range"); + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); } if (is_string()) @@ -4088,7 +4720,7 @@ class basic_json default: { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4109,7 +4741,7 @@ class basic_json @post References and iterators to the erased elements are invalidated. Other references and iterators are not affected. - @throw std::domain_error when called on a type other than JSON object; + @throw type_error.307 when called on a type other than JSON object; example: `"cannot use erase() with null"` @complexity `log(size()) + count(key)` @@ -4131,10 +4763,8 @@ class basic_json { return m_value.object->erase(key); } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } /*! @@ -4144,9 +4774,9 @@ class basic_json @param[in] idx index of the element to remove - @throw std::domain_error when called on a type other than JSON array; + @throw type_error.307 when called on a type other than JSON object; example: `"cannot use erase() with null"` - @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + @throw out_of_range.401 when `idx >= size()`; example: `"array index 17 is out of range"` @complexity Linear in distance between @a idx and the end of the container. @@ -4168,14 +4798,14 @@ class basic_json { if (idx >= size()) { - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } m_value.array->erase(m_value.array->begin() + static_cast(idx)); } else { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4877,7 +5507,7 @@ class basic_json @param[in] val the value to add to the JSON array - @throw std::domain_error when called on a type other than JSON array or + @throw type_error.308 when called on a type other than JSON array or null; example: `"cannot use push_back() with number"` @complexity Amortized constant. @@ -4893,7 +5523,7 @@ class basic_json // push_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an array @@ -4929,7 +5559,7 @@ class basic_json // push_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an array @@ -4963,7 +5593,7 @@ class basic_json @param[in] val the value to add to the JSON object - @throw std::domain_error when called on a type other than JSON object or + @throw type_error.308 when called on a type other than JSON object or null; example: `"cannot use push_back() with number"` @complexity Logarithmic in the size of the container, O(log(`size()`)). @@ -4979,7 +5609,7 @@ class basic_json // push_back only works for null objects or objects if (not(is_null() or is_object())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an object @@ -5017,7 +5647,7 @@ class basic_json @ref push_back(const typename object_t::value_type&). Otherwise, @a init is converted to a JSON value and added using @ref push_back(basic_json&&). - @param init an initializer list + @param[in] init an initializer list @complexity Linear in the size of the initializer list @a init. @@ -5062,7 +5692,7 @@ class basic_json @param[in] args arguments to forward to a constructor of @ref basic_json @tparam Args compatible types to create a @ref basic_json object - @throw std::domain_error when called on a type other than JSON array or + @throw type_error.311 when called on a type other than JSON array or null; example: `"cannot use emplace_back() with number"` @complexity Amortized constant. @@ -5079,7 +5709,7 @@ class basic_json // emplace_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use emplace_back() with " + type_name()); + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + type_name())); } // transform null object into an array @@ -5097,8 +5727,8 @@ class basic_json /*! @brief add an object to an object if key does not exist - Inserts a new element into a JSON object constructed in-place with the given - @a args if there is no element with the key in the container. If the + Inserts a new element into a JSON object constructed in-place with the + given @a args if there is no element with the key in the container. If the function is called on a JSON null value, an empty object is created before appending the value created from @a args. @@ -5109,7 +5739,7 @@ class basic_json already-existing element if no insertion happened, and a bool denoting whether the insertion took place. - @throw std::domain_error when called on a type other than JSON object or + @throw type_error.311 when called on a type other than JSON object or null; example: `"cannot use emplace() with number"` @complexity Logarithmic in the size of the container, O(log(`size()`)). @@ -5127,7 +5757,7 @@ class basic_json // emplace only works for null objects or arrays if (not(is_null() or is_object())) { - throw std::domain_error("cannot use emplace() with " + type_name()); + JSON_THROW(type_error::create(311, "cannot use emplace() with " + type_name())); } // transform null object into an object @@ -5158,13 +5788,13 @@ class basic_json @param[in] val element to insert @return iterator pointing to the inserted @a val. - @throw std::domain_error if called on JSON values other than arrays; + @throw type_error.309 if called on JSON values other than arrays; example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` - @complexity Constant plus linear in the distance between pos and end of the - container. + @complexity Constant plus linear in the distance between @a pos and end of + the container. @liveexample{The example shows how `insert()` is used.,insert} @@ -5178,7 +5808,7 @@ class basic_json // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5186,10 +5816,8 @@ class basic_json result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); return result; } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } /*! @@ -5213,10 +5841,10 @@ class basic_json @return iterator pointing to the first element inserted, or @a pos if `cnt==0` - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` @complexity Linear in @a cnt plus linear in the distance between @a pos and end of the container. @@ -5233,7 +5861,7 @@ class basic_json // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5241,10 +5869,8 @@ class basic_json result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); return result; } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } /*! @@ -5257,13 +5883,13 @@ class basic_json @param[in] first begin of the range of elements to insert @param[in] last end of the range of elements to insert - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - @throw std::domain_error if @a first and @a last do not belong to the same - JSON value; example: `"iterators do not fit"` - @throw std::domain_error if @a first or @a last are iterators into + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + @throw invalid_iterator.211 if @a first or @a last are iterators into container for which insert is called; example: `"passed iterators may not belong to container"` @@ -5282,24 +5908,24 @@ class basic_json // insert only works for arrays if (not is_array()) { - throw std::domain_error("cannot use insert() with " + type_name()); + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // check if range iterators belong to the same JSON object if (first.m_object != last.m_object) { - throw std::domain_error("iterators do not fit"); + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); } if (first.m_object == this or last.m_object == this) { - throw std::domain_error("passed iterators may not belong to container"); + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container")); } // insert to array and return iterator @@ -5320,10 +5946,10 @@ class basic_json the end() iterator @param[in] ilist initializer list to insert the values from - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` @return iterator pointing to the first element inserted, or @a pos if `ilist` is empty @@ -5340,13 +5966,13 @@ class basic_json // insert only works for arrays if (not is_array()) { - throw std::domain_error("cannot use insert() with " + type_name()); + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5355,6 +5981,52 @@ class basic_json return result; } + /*! + @brief inserts elements + + Inserts elements from range `[first, last)`. + + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw type_error.309 if called on JSON values other than objects; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if iterator @a first or @a last does does not + point to an object; example: `"iterators first and last must point to + objects"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + + @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number + of elements to insert. + + @liveexample{The example shows how `insert()` is used.,insert__range_object} + + @since version 3.0.0 + */ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (not is_object()) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); + } + + // passed iterators must belong to objects + if (not first.m_object->is_object() or not first.m_object->is_object()) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + /*! @brief exchanges the values @@ -5394,7 +6066,7 @@ class basic_json @param[in,out] other array to exchange the contents with - @throw std::domain_error when JSON value is not an array; example: `"cannot + @throw type_error.310 when JSON value is not an array; example: `"cannot use swap() with string"` @complexity Constant. @@ -5413,7 +6085,7 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } @@ -5427,7 +6099,7 @@ class basic_json @param[in,out] other object to exchange the contents with - @throw std::domain_error when JSON value is not an object; example: + @throw type_error.310 when JSON value is not an object; example: `"cannot use swap() with string"` @complexity Constant. @@ -5446,7 +6118,7 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } @@ -5460,7 +6132,7 @@ class basic_json @param[in,out] other string to exchange the contents with - @throw std::domain_error when JSON value is not a string; example: `"cannot + @throw type_error.310 when JSON value is not a string; example: `"cannot use swap() with boolean"` @complexity Constant. @@ -5479,13 +6151,13 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } /// @} - + public: ////////////////////////////////////////// // lexicographical comparison operators // ////////////////////////////////////////// @@ -5493,64 +6165,198 @@ class basic_json /// @name lexicographical comparison operators /// @{ - private: /*! - @brief comparison operator for JSON types + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same according to their respective + `operator==`. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. Note than two NaN values are always + treated as unequal. + - Two JSON null values are equal. + + @note NaN values never compare equal to themselves or to other NaN values. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs == basic_json(rhs)); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) == rhs); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal - Returns an ordering that is similar to Python: - - order: null < boolean < number < object < array < string - - furthermore, each type is not smaller than itself + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} @since version 1.0.0 */ - friend bool operator<(const value_t lhs, const value_t rhs) noexcept + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept { - static constexpr std::array order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float - } - }; + return not (lhs == rhs); + } - // discarded values are not comparable - if (lhs == value_t::discarded or rhs == value_t::discarded) - { - return false; - } + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs != basic_json(rhs)); + } - return order[static_cast(lhs)] < order[static_cast(rhs)]; + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) != rhs); } - public: /*! - @brief comparison: equal + @brief comparison: less than - Compares two JSON values for equality according to the following rules: - - Two JSON values are equal if (1) they are from the same type and (2) - their stored values are the same. + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. - Integer and floating-point numbers are automatically converted before - comparison. Floating-point numbers are compared indirectly: two - floating-point numbers `f1` and `f2` are considered equal if neither - `f1 > f2` nor `f2 > f1` holds. - - Two JSON null values are equal. + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). @param[in] lhs first JSON value to consider @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are equal + @return whether @a lhs is less than @a rhs @complexity Linear. @liveexample{The example demonstrates comparing several JSON - types.,operator__equal} + types.,operator__less} @since version 1.0.0 */ - friend bool operator==(const_reference lhs, const_reference rhs) noexcept + friend bool operator<(const_reference lhs, const_reference rhs) noexcept { const auto lhs_type = lhs.type(); const auto rhs_type = rhs.type(); @@ -5561,35 +6367,35 @@ class basic_json { case value_t::array: { - return *lhs.m_value.array == *rhs.m_value.array; + return *lhs.m_value.array < *rhs.m_value.array; } case value_t::object: { - return *lhs.m_value.object == *rhs.m_value.object; + return *lhs.m_value.object < *rhs.m_value.object; } case value_t::null: { - return true; + return false; } case value_t::string: { - return *lhs.m_value.string == *rhs.m_value.string; + return *lhs.m_value.string < *rhs.m_value.string; } case value_t::boolean: { - return lhs.m_value.boolean == rhs.m_value.boolean; + return lhs.m_value.boolean < rhs.m_value.boolean; } case value_t::number_integer: { - return lhs.m_value.number_integer == rhs.m_value.number_integer; + return lhs.m_value.number_integer < rhs.m_value.number_integer; } case value_t::number_unsigned: { - return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; } case value_t::number_float: { - return lhs.m_value.number_float == rhs.m_value.number_float; + return lhs.m_value.number_float < rhs.m_value.number_float; } default: { @@ -5599,295 +6405,786 @@ class basic_json } else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) { - return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); } else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) { - return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) { - return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) { - return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; } - return false; + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); } /*! - @brief comparison: equal + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs < basic_json(rhs)); + } + + /*! + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) < rhs); + } - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `v.is_null()`. + /*! + @brief comparison: less than or equal - @param[in] v JSON value to consider - @return whether @a v is null + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. - @complexity Constant. + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. - @liveexample{The example compares several JSON types to the null pointer. - ,operator__equal__nullptr_t} + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} @since version 1.0.0 */ - friend bool operator==(const_reference v, std::nullptr_t) noexcept + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept { - return v.is_null(); + return not (rhs < lhs); } /*! - @brief comparison: equal - @copydoc operator==(const_reference, std::nullptr_t) + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) */ - friend bool operator==(std::nullptr_t, const_reference v) noexcept + template::value, int>::type = 0> + friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept { - return v.is_null(); + return (lhs <= basic_json(rhs)); } /*! - @brief comparison: not equal + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) <= rhs); + } - Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. @param[in] lhs first JSON value to consider @param[in] rhs second JSON value to consider - @return whether the values @a lhs and @a rhs are not equal + @return whether @a lhs is greater than to @a rhs @complexity Linear. @liveexample{The example demonstrates comparing several JSON - types.,operator__notequal} + types.,operator__lessequal} @since version 1.0.0 */ - friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + friend bool operator>(const_reference lhs, const_reference rhs) noexcept { - return not (lhs == rhs); + return not (lhs <= rhs); } /*! - @brief comparison: not equal + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs > basic_json(rhs)); + } + + /*! + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) > rhs); + } - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `not v.is_null()`. + /*! + @brief comparison: greater than or equal - @param[in] v JSON value to consider - @return whether @a v is not null + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. - @complexity Constant. + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. - @liveexample{The example compares several JSON types to the null pointer. - ,operator__notequal__nullptr_t} + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} @since version 1.0.0 */ - friend bool operator!=(const_reference v, std::nullptr_t) noexcept + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept { - return not v.is_null(); + return not (lhs < rhs); } /*! - @brief comparison: not equal - @copydoc operator!=(const_reference, std::nullptr_t) + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) */ - friend bool operator!=(std::nullptr_t, const_reference v) noexcept + template::value, int>::type = 0> + friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept { - return not v.is_null(); + return (lhs >= basic_json(rhs)); } /*! - @brief comparison: less than + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) >= rhs); + } - Compares whether one JSON value @a lhs is less than another JSON value @a - rhs according to the following rules: - - If @a lhs and @a rhs have the same type, the values are compared using - the default `<` operator. - - Integer and floating-point numbers are automatically converted before - comparison - - In case @a lhs and @a rhs have different types, the values are ignored - and the order of the types is considered, see - @ref operator<(const value_t, const value_t). + /// @} - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than @a rhs - @complexity Linear. + /////////////////// + // serialization // + /////////////////// - @liveexample{The example demonstrates comparing several JSON - types.,operator__less} + /// @name serialization + /// @{ - @since version 1.0.0 + private: + /*! + @brief wrapper around the serialization functions */ - friend bool operator<(const_reference lhs, const_reference rhs) noexcept + class serializer { - const auto lhs_type = lhs.type(); - const auto rhs_type = rhs.type(); + private: + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; - if (lhs_type == rhs_type) + public: + /*! + @param[in] s output stream to serialize to + */ + serializer(std::ostream& s) + : o(s), loc(std::localeconv()), + thousands_sep(!loc->thousands_sep ? '\0' : loc->thousands_sep[0]), + decimal_point(!loc->decimal_point ? '\0' : loc->decimal_point[0]) + {} + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and + organizes the serialization internally. The indentation level is + propagated as additional parameter. In case of arrays and objects, the + function is called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const basic_json& val, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) { - switch (lhs_type) + switch (val.m_type) { - case value_t::array: - { - return *lhs.m_value.array < *rhs.m_value.array; - } case value_t::object: { - return *lhs.m_value.object < *rhs.m_value.object; + if (val.m_value.object->empty()) + { + o.write("{}", 2); + return; + } + + if (pretty_print) + { + o.write("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (indent_string.size() < new_indent) + { + indent_string.resize(new_indent, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o.write(indent_string.c_str(), static_cast(new_indent)); + o.put('\"'); + dump_escaped(i->first); + o.write("\": ", 3); + dump(i->second, true, indent_step, new_indent); + o.write(",\n", 2); + } + + // last element + assert(i != val.m_value.object->cend()); + o.write(indent_string.c_str(), static_cast(new_indent)); + o.put('\"'); + dump_escaped(i->first); + o.write("\": ", 3); + dump(i->second, true, indent_step, new_indent); + + o.put('\n'); + o.write(indent_string.c_str(), static_cast(current_indent)); + o.put('}'); + } + else + { + o.put('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o.put('\"'); + dump_escaped(i->first); + o.write("\":", 2); + dump(i->second, false, indent_step, current_indent); + o.put(','); + } + + // last element + assert(i != val.m_value.object->cend()); + o.put('\"'); + dump_escaped(i->first); + o.write("\":", 2); + dump(i->second, false, indent_step, current_indent); + + o.put('}'); + } + + return; } - case value_t::null: + + case value_t::array: { - return false; + if (val.m_value.array->empty()) + { + o.write("[]", 2); + return; + } + + if (pretty_print) + { + o.write("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (indent_string.size() < new_indent) + { + indent_string.resize(new_indent, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i) + { + o.write(indent_string.c_str(), static_cast(new_indent)); + dump(*i, true, indent_step, new_indent); + o.write(",\n", 2); + } + + // last element + assert(not val.m_value.array->empty()); + o.write(indent_string.c_str(), static_cast(new_indent)); + dump(val.m_value.array->back(), true, indent_step, new_indent); + + o.put('\n'); + o.write(indent_string.c_str(), static_cast(current_indent)); + o.put(']'); + } + else + { + o.put('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, indent_step, current_indent); + o.put(','); + } + + // last element + assert(not val.m_value.array->empty()); + dump(val.m_value.array->back(), false, indent_step, current_indent); + + o.put(']'); + } + + return; } + case value_t::string: { - return *lhs.m_value.string < *rhs.m_value.string; + o.put('\"'); + dump_escaped(*val.m_value.string); + o.put('\"'); + return; } + case value_t::boolean: { - return lhs.m_value.boolean < rhs.m_value.boolean; + if (val.m_value.boolean) + { + o.write("true", 4); + } + else + { + o.write("false", 5); + } + return; } + case value_t::number_integer: { - return lhs.m_value.number_integer < rhs.m_value.number_integer; + dump_integer(val.m_value.number_integer); + return; } + case value_t::number_unsigned: { - return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + dump_integer(val.m_value.number_unsigned); + return; } + case value_t::number_float: { - return lhs.m_value.number_float < rhs.m_value.number_float; + dump_float(val.m_value.number_float); + return; } - default: + + case value_t::discarded: { - return false; + o.write("", 11); + return; + } + + case value_t::null: + { + o.write("null", 4); + return; } } } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); - } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) - { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; - } - else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); - } - else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) - { - return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + + private: + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x0b: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1a: + case 0x1b: + case 0x1c: + case 0x1d: + case 0x1e: + case 0x1f: + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + + default: + { + return res; + } + } + }); } - else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence + of an escape character (backslash) and another character and other + control characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s) const { - return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + const auto space = extra_space(s); + if (space == 0) + { + o.write(s.c_str(), static_cast(s.size())); + return; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x0b: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1a: + case 0x1b: + case 0x1c: + case 0x1d: + case 0x1e: + case 0x1f: + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + break; + } + + default: + { + // all other characters are added as-is + result[pos++] = c; + break; + } + } + } + + assert(pos == s.size() + space); + o.write(result.c_str(), static_cast(result.size())); } - // We only reach this line if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - return operator<(lhs_type, rhs_type); - } + /*! + @brief dump an integer - /*! - @brief comparison: less than or equal + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. - Compares whether one JSON value @a lhs is less than or equal to another - JSON value by calculating `not (rhs < lhs)`. + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template::value or + std::is_same::value, int> = 0> + void dump_integer(NumberType x) + { + // special case for "0" + if (x == 0) + { + o.put('0'); + return; + } - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is less than or equal to @a rhs + const bool is_negative = x < 0; + size_t i = 0; - @complexity Linear. + // spare 1 byte for '\0' + while (x != 0 and i < number_buffer.size() - 1) + { + const auto digit = std::labs(static_cast(x % 10)); + number_buffer[i++] = static_cast('0' + digit); + x /= 10; + } - @liveexample{The example demonstrates comparing several JSON - types.,operator__greater} + // make sure the number has been processed completely + assert(x == 0); - @since version 1.0.0 - */ - friend bool operator<=(const_reference lhs, const_reference rhs) noexcept - { - return not (rhs < lhs); - } + if (is_negative) + { + // make sure there is capacity for the '-' + assert(i < number_buffer.size() - 2); + number_buffer[i++] = '-'; + } - /*! - @brief comparison: greater than + std::reverse(number_buffer.begin(), number_buffer.begin() + i); + o.write(number_buffer.data(), static_cast(i)); + } - Compares whether one JSON value @a lhs is greater than another - JSON value by calculating `not (lhs <= rhs)`. + /*! + @brief dump a floating-point number - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than to @a rhs + Dump a given floating-point number to output stream @a o. Works + internally with @a number_buffer. - @complexity Linear. + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (not std::isfinite(x) or std::isnan(x)) + { + o.write("null", 4); + return; + } - @liveexample{The example demonstrates comparing several JSON - types.,operator__lessequal} + // special case for 0.0 and -0.0 + if (x == 0) + { + if (std::signbit(x)) + { + o.write("-0.0", 4); + } + else + { + o.write("0.0", 3); + } + return; + } - @since version 1.0.0 - */ - friend bool operator>(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs <= rhs); - } + // get number of digits for a text -> float -> text round-trip + static constexpr auto d = std::numeric_limits::digits10; - /*! - @brief comparison: greater than or equal + // the actual conversion + std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), + "%.*g", d, x); - Compares whether one JSON value @a lhs is greater than or equal to another - JSON value by calculating `not (lhs < rhs)`. + // negative value indicates an error + assert(len > 0); + // check if buffer was large enough + assert(static_cast(len) < number_buffer.size()); - @param[in] lhs first JSON value to consider - @param[in] rhs second JSON value to consider - @return whether @a lhs is greater than or equal to @a rhs + // erase thousands separator + if (thousands_sep != '\0') + { + const auto end = std::remove(number_buffer.begin(), + number_buffer.begin() + len, + thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + assert((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } - @complexity Linear. + // convert decimal point to '.' + if (decimal_point != '\0' and decimal_point != '.') + { + for (auto& c : number_buffer) + { + if (c == decimal_point) + { + c = '.'; + break; + } + } + } - @liveexample{The example demonstrates comparing several JSON - types.,operator__greaterequal} + o.write(number_buffer.data(), static_cast(len)); - @since version 1.0.0 - */ - friend bool operator>=(const_reference lhs, const_reference rhs) noexcept - { - return not (lhs < rhs); - } + // determine if need to append ".0" + const bool value_is_int_like = std::none_of(number_buffer.begin(), + number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' or c == 'e'; + }); - /// @} + if (value_is_int_like) + { + o.write(".0", 2); + } + } + private: + /// the output of the serializer + std::ostream& o; - /////////////////// - // serialization // - /////////////////// + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; - /// @name serialization - /// @{ + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// the indentation string + string_t indent_string = string_t(512, ' '); + }; + public: /*! @brief serialize to stream @@ -5898,10 +7195,6 @@ class basic_json `std::setw(4)` on @a o sets the indentation level to `4` and the serialization result is the same as calling `dump(4)`. - @note During serializaion, the locale and the precision of the output - stream @a o are changed. The original values are restored when the - function returns. - @param[in,out] o stream to serialize to @param[in] j JSON value to serialize @@ -5923,29 +7216,20 @@ class basic_json // reset width to 0 for subsequent calls to this stream o.width(0); - // fix locale problems - const auto old_locale = o.imbue(std::locale::classic()); - // set precision - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - const auto old_precision = o.precision(std::numeric_limits::digits10); - // do the actual serialization - j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale and precision - o.imbue(old_locale); - o.precision(old_precision); + serializer s(o); + s.dump(j, pretty_print, static_cast(indentation)); return o; } /*! @brief serialize to stream - @copydoc operator<<(std::ostream&, const basic_json&) + @deprecated This stream operator is deprecated and will be removed in a + future version of the library. Please use + @ref std::ostream& operator<<(std::ostream&, const basic_json&) + instead; that is, replace calls like `j >> o;` with `o << j;`. */ + JSON_DEPRECATED friend std::ostream& operator>>(const basic_json& j, std::ostream& o) { return o << j; @@ -5977,6 +7261,11 @@ class basic_json @return result of the deserialization + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6007,6 +7296,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6043,6 +7336,11 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @throw parse_error.111 if input stream is in a bad state + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6102,6 +7400,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6122,7 +7424,7 @@ class basic_json { // assertion to check that the iterator range is indeed contiguous, // see http://stackoverflow.com/a/35008842/266378 for more discussion - assert(std::accumulate(first, last, std::make_pair(true, 0), + assert(std::accumulate(first, last, std::pair(true, 0), [&first](std::pair res, decltype(*first) val) { res.first &= (val == *(std::next(std::addressof(*first), res.second++))); @@ -6172,6 +7474,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6192,8 +7498,22 @@ class basic_json static basic_json parse(const ContiguousContainer& c, const parser_callback_t cb = nullptr) { - // delegate the call to the iterator-range parse overload - return parse(std::begin(c), std::end(c), cb); + // delegate the call to the iterator-range parse overload + return parse(std::begin(c), std::end(c), cb); + } + + /*! + @brief deserialize from stream + @deprecated This stream operator is deprecated and will be removed in a + future version of the library. Please use + @ref std::istream& operator>>(std::istream&, basic_json&) + instead; that is, replace calls like `j << i;` with `i >> j;`. + */ + JSON_DEPRECATED + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; } /*! @@ -6204,7 +7524,10 @@ class basic_json @param[in,out] i input stream to read a serialized JSON value from @param[in,out] j JSON value to write the deserialized input to - @throw std::invalid_argument in case of parse errors + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @throw parse_error.111 if input stream is in a bad state @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. @@ -6219,16 +7542,6 @@ class basic_json @since version 1.0.0 */ - friend std::istream& operator<<(basic_json& j, std::istream& i) - { - j = parser(i).parse(); - return i; - } - - /*! - @brief deserialize from stream - @copydoc operator<<(basic_json&, std::istream&) - */ friend std::istream& operator>>(std::istream& i, basic_json& j) { j = parser(i).parse(); @@ -6245,6 +7558,11 @@ class basic_json /// @{ private: + /*! + @note Some code in the switch cases has been copied, because otherwise + copilers would complain about implicit fallthrough and there is no + portable attribute to mute such warnings. + */ template static void add_to_vector(std::vector& vec, size_t bytes, const T number) { @@ -6254,24 +7572,31 @@ class basic_json { case 8: { - vec.push_back(static_cast((number >> 070) & 0xff)); - vec.push_back(static_cast((number >> 060) & 0xff)); - vec.push_back(static_cast((number >> 050) & 0xff)); - vec.push_back(static_cast((number >> 040) & 0xff)); - // intentional fall-through + vec.push_back(static_cast((static_cast(number) >> 070) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 060) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 050) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 040) & 0xff)); + vec.push_back(static_cast((number >> 030) & 0xff)); + vec.push_back(static_cast((number >> 020) & 0xff)); + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; } case 4: { vec.push_back(static_cast((number >> 030) & 0xff)); vec.push_back(static_cast((number >> 020) & 0xff)); - // intentional fall-through + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; } case 2: { vec.push_back(static_cast((number >> 010) & 0xff)); - // intentional fall-through + vec.push_back(static_cast(number & 0xff)); + break; } case 1: @@ -6296,7 +7621,7 @@ class basic_json @tparam T the integral return type - @throw std::out_of_range if there are less than sizeof(T)+1 bytes in the + @throw parse_error.110 if there are less than sizeof(T)+1 bytes in the vector @a vec to read In the for loop, the bytes from the vector are copied in reverse order into @@ -6321,13 +7646,11 @@ class basic_json template static T get_from_vector(const std::vector& vec, const size_t current_index) { - if (current_index + sizeof(T) + 1 > vec.size()) - { - throw std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector"); - } + // check if we can read sizeof(T) bytes starting the next index + check_length(vec.size(), sizeof(T), current_index + 1); T result; - uint8_t* ptr = reinterpret_cast(&result); + auto* ptr = reinterpret_cast(&result); for (size_t i = 0; i < sizeof(T); ++i) { *ptr++ = vec[current_index + sizeof(T) - i]; @@ -6368,32 +7691,33 @@ class basic_json if (j.m_value.number_integer >= 0) { // MessagePack does not differentiate between positive - // signed integers and unsigned integers. Therefore, we used - // the code from the value_t::number_unsigned case here. + // signed integers and unsigned integers. Therefore, we + // used the code from the value_t::number_unsigned case + // here. if (j.m_value.number_unsigned < 128) { // positive fixnum add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT8_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 v.push_back(0xcc); add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT16_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 v.push_back(0xcd); add_to_vector(v, 2, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT32_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 v.push_back(0xce); add_to_vector(v, 4, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT64_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 v.push_back(0xcf); @@ -6407,25 +7731,25 @@ class basic_json // negative fixnum add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT8_MIN and j.m_value.number_integer <= INT8_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 v.push_back(0xd0); add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT16_MIN and j.m_value.number_integer <= INT16_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 v.push_back(0xd1); add_to_vector(v, 2, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT32_MIN and j.m_value.number_integer <= INT32_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 v.push_back(0xd2); add_to_vector(v, 4, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT64_MIN and j.m_value.number_integer <= INT64_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 v.push_back(0xd3); @@ -6442,25 +7766,25 @@ class basic_json // positive fixnum add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT8_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 v.push_back(0xcc); add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT16_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 v.push_back(0xcd); add_to_vector(v, 2, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT32_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 v.push_back(0xce); add_to_vector(v, 4, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT64_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 v.push_back(0xcf); @@ -6473,7 +7797,7 @@ class basic_json { // float 64 v.push_back(0xcb); - const uint8_t* helper = reinterpret_cast(&(j.m_value.number_float)); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); for (size_t i = 0; i < 8; ++i) { v.push_back(helper[7 - i]); @@ -6617,19 +7941,19 @@ class basic_json { add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT8_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x18); // one-byte uint8_t add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT16_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x19); // two-byte uint16_t add_to_vector(v, 2, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT32_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x1a); // four-byte uint32_t @@ -6644,26 +7968,26 @@ class basic_json } else { - // The conversions below encode the sign in the first byte, - // and the value is converted to a positive number. + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. const auto positive_number = -1 - j.m_value.number_integer; if (j.m_value.number_integer >= -24) { v.push_back(static_cast(0x20 + positive_number)); } - else if (positive_number <= UINT8_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 8 v.push_back(0x38); add_to_vector(v, 1, positive_number); } - else if (positive_number <= UINT16_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 16 v.push_back(0x39); add_to_vector(v, 2, positive_number); } - else if (positive_number <= UINT32_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 32 v.push_back(0x3a); @@ -6716,7 +8040,7 @@ class basic_json { // Double-Precision Float v.push_back(0xfb); - const uint8_t* helper = reinterpret_cast(&(j.m_value.number_float)); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); for (size_t i = 0; i < 8; ++i) { v.push_back(helper[7 - i]); @@ -6729,7 +8053,7 @@ class basic_json const auto N = j.m_value.string->size(); if (N <= 0x17) { - v.push_back(0x60 + N); // 1 byte for string + size + v.push_back(static_cast(0x60 + N)); // 1 byte for string + size } else if (N <= 0xff) { @@ -6765,7 +8089,7 @@ class basic_json const auto N = j.m_value.array->size(); if (N <= 0x17) { - v.push_back(0x80 + N); // 1 byte for array + size + v.push_back(static_cast(0x80 + N)); // 1 byte for array + size } else if (N <= 0xff) { @@ -6803,7 +8127,7 @@ class basic_json const auto N = j.m_value.object->size(); if (N <= 0x17) { - v.push_back(0xa0 + N); // 1 byte for object + size + v.push_back(static_cast(0xa0 + N)); // 1 byte for object + size } else if (N <= 0xff) { @@ -6850,12 +8174,12 @@ class basic_json To secure the access to the byte vector during CBOR/MessagePack deserialization, bytes are copied from the vector into buffers. This - function checks if the number of bytes to copy (@a len) does not exceed the - size @s size of the vector. Additionally, an @a offset is given from where - to start reading the bytes. + function checks if the number of bytes to copy (@a len) does not exceed + the size @s size of the vector. Additionally, an @a offset is given from + where to start reading the bytes. - This function checks whether reading the bytes is safe; that is, offset is a - valid index in the vector, offset+len + This function checks whether reading the bytes is safe; that is, offset is + a valid index in the vector, offset+len @param[in] size size of the byte vector @param[in] len number of bytes to read @@ -6872,20 +8196,81 @@ class basic_json // simple case: requested length is greater than the vector's length if (len > size or offset > size) { - throw std::out_of_range("len out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } // second case: adding offset would result in overflow - if ((size > (std::numeric_limits::max() - offset))) + if ((size > ((std::numeric_limits::max)() - offset))) { - throw std::out_of_range("len+offset out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } // last case: reading past the end of the vector if (len + offset > size) { - throw std::out_of_range("len+offset out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); + } + } + + /*! + @brief check if the next byte belongs to a string + + While parsing a map, the keys must be strings. This function checks if the + current byte is one of the start bytes for a string in MessagePack: + + - 0xa0 - 0xbf: fixstr + - 0xd9: str 8 + - 0xda: str 16 + - 0xdb: str 32 + + @param[in] v MessagePack serialization + @param[in] idx byte index in @a v to check for a string + + @throw parse_error.113 if `v[idx]` does not belong to a string + */ + static void msgpack_expect_string(const std::vector& v, size_t idx) + { + check_length(v.size(), 1, idx); + + const auto byte = v[idx]; + if ((byte >= 0xa0 and byte <= 0xbf) or (byte >= 0xd9 and byte <= 0xdb)) + { + return; + } + + std::stringstream ss; + ss << std::hex << static_cast(v[idx]); + JSON_THROW(parse_error::create(113, idx + 1, "expected a MessagePack string; last byte: 0x" + ss.str())); + } + + /*! + @brief check if the next byte belongs to a string + + While parsing a map, the keys must be strings. This function checks if the + current byte is one of the start bytes for a string in CBOR: + + - 0x60 - 0x77: fixed length + - 0x78 - 0x7b: variable length + - 0x7f: indefinity length + + @param[in] v CBOR serialization + @param[in] idx byte index in @a v to check for a string + + @throw parse_error.113 if `v[idx]` does not belong to a string + */ + static void cbor_expect_string(const std::vector& v, size_t idx) + { + check_length(v.size(), 1, idx); + + const auto byte = v[idx]; + if ((byte >= 0x60 and byte <= 0x7b) or byte == 0x7f) + { + return; } + + std::stringstream ss; + ss << std::hex << static_cast(v[idx]); + JSON_THROW(parse_error::create(113, idx + 1, "expected a CBOR string; last byte: 0x" + ss.str())); } /*! @@ -6896,32 +8281,34 @@ class basic_json @return deserialized JSON value - @throw std::invalid_argument if unsupported features from MessagePack were + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from MessagePack were used in the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.113 if a string was expected as map key, but not found @sa https://github.com/msgpack/msgpack/blob/master/spec.md */ static basic_json from_msgpack_internal(const std::vector& v, size_t& idx) { - // make sure reading 1 byte is safe - check_length(v.size(), 1, idx); - // store and increment index const size_t current_idx = idx++; + // make sure reading 1 byte is safe + check_length(v.size(), 1, current_idx); + if (v[current_idx] <= 0xbf) { if (v[current_idx] <= 0x7f) // positive fixint { return v[current_idx]; } - else if (v[current_idx] <= 0x8f) // fixmap + if (v[current_idx] <= 0x8f) // fixmap { basic_json result = value_t::object; const size_t len = v[current_idx] & 0x0f; for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -6972,8 +8359,8 @@ class basic_json case 0xca: // float 32 { // copy bytes in reverse order into the double variable - check_length(v.size(), sizeof(float), 1); float res; + check_length(v.size(), sizeof(float), current_idx + 1); for (size_t byte = 0; byte < sizeof(float); ++byte) { reinterpret_cast(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte]; @@ -6985,8 +8372,8 @@ class basic_json case 0xcb: // float 64 { // copy bytes in reverse order into the double variable - check_length(v.size(), sizeof(double), 1); double res; + check_length(v.size(), sizeof(double), current_idx + 1); for (size_t byte = 0; byte < sizeof(double); ++byte) { reinterpret_cast(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte]; @@ -7101,6 +8488,7 @@ class basic_json idx += 2; // skip 2 size bytes for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -7114,6 +8502,7 @@ class basic_json idx += 4; // skip 4 size bytes for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -7122,7 +8511,9 @@ class basic_json default: { - throw std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx]))); + std::stringstream ss; + ss << std::hex << static_cast(v[current_idx]); + JSON_THROW(parse_error::create(112, current_idx + 1, "error reading MessagePack; last byte: 0x" + ss.str())); } } } @@ -7136,9 +8527,10 @@ class basic_json @return deserialized JSON value - @throw std::invalid_argument if unsupported features from CBOR were used in - the given vector @a v or if the input is not valid CBOR - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from CBOR were + used in the given vector @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found @sa https://tools.ietf.org/html/rfc7049 */ @@ -7147,7 +8539,10 @@ class basic_json // store and increment index const size_t current_idx = idx++; - switch (v.at(current_idx)) + // make sure reading 1 byte is safe + check_length(v.size(), 1, current_idx); + + switch (v[current_idx]) { // Integer 0x00..0x17 (0..23) case 0x00: @@ -7328,7 +8723,7 @@ class basic_json case 0x7f: // UTF-8 string (indefinite length) { std::string result; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { string_t s = from_cbor_internal(v, idx); result += s; @@ -7424,7 +8819,7 @@ class basic_json case 0x9f: // array (indefinite length) { basic_json result = value_t::array; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { result.push_back(from_cbor_internal(v, idx)); } @@ -7463,6 +8858,7 @@ class basic_json const auto len = static_cast(v[current_idx] - 0xa0); for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7476,6 +8872,7 @@ class basic_json idx += 1; // skip 1 size byte for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7489,6 +8886,7 @@ class basic_json idx += 2; // skip 2 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7502,6 +8900,7 @@ class basic_json idx += 4; // skip 4 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7515,6 +8914,7 @@ class basic_json idx += 8; // skip 8 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7524,8 +8924,9 @@ class basic_json case 0xbf: // map (indefinite length) { basic_json result = value_t::object; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7551,7 +8952,6 @@ class basic_json case 0xf9: // Half-Precision Float (two-byte IEEE 754) { - check_length(v.size(), 2, 1); idx += 2; // skip two content bytes // code from RFC 7049, Appendix D, Figure 3: @@ -7561,6 +8961,7 @@ class basic_json // include at least decoding support for them even without such // support. An example of a small decoder for half-precision // floating-point numbers in the C language is shown in Fig. 3. + check_length(v.size(), 2, current_idx + 1); const int half = (v[current_idx + 1] << 8) + v[current_idx + 2]; const int exp = (half >> 10) & 0x1f; const int mant = half & 0x3ff; @@ -7575,16 +8976,18 @@ class basic_json } else { - val = mant == 0 ? INFINITY : NAN; + val = mant == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); } - return half & 0x8000 ? -val : val; + return (half & 0x8000) != 0 ? -val : val; } case 0xfa: // Single-Precision Float (four-byte IEEE 754) { // copy bytes in reverse order into the float variable - check_length(v.size(), sizeof(float), 1); float res; + check_length(v.size(), sizeof(float), current_idx + 1); for (size_t byte = 0; byte < sizeof(float); ++byte) { reinterpret_cast(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte]; @@ -7595,9 +8998,9 @@ class basic_json case 0xfb: // Double-Precision Float (eight-byte IEEE 754) { - check_length(v.size(), sizeof(double), 1); // copy bytes in reverse order into the double variable double res; + check_length(v.size(), sizeof(double), current_idx + 1); for (size_t byte = 0; byte < sizeof(double); ++byte) { reinterpret_cast(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte]; @@ -7608,7 +9011,9 @@ class basic_json default: // anything else (0xFF is handled inside the other types) { - throw std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx]))); + std::stringstream ss; + ss << std::hex << static_cast(v[current_idx]); + JSON_THROW(parse_error::create(112, current_idx + 1, "error reading CBOR; last byte: 0x" + ss.str())); } } } @@ -7621,6 +9026,58 @@ class basic_json serialization format. MessagePack is a binary serialization format which aims to be more compact than JSON itself, yet more efficient to parse. + The library uses the following mapping from JSON values types to + MessagePack types according to the MessagePack specification: + + JSON value type | value/range | MessagePack type | first byte + --------------- | --------------------------------- | ---------------- | ---------- + null | `null` | nil | 0xc0 + boolean | `true` | true | 0xc3 + boolean | `false` | false | 0xc2 + number_integer | -9223372036854775808..-2147483649 | int64 | 0xd3 + number_integer | -2147483648..-32769 | int32 | 0xd2 + number_integer | -32768..-129 | int16 | 0xd1 + number_integer | -128..-33 | int8 | 0xd0 + number_integer | -32..-1 | negative fixint | 0xe0..0xff + number_integer | 0..127 | positive fixint | 0x00..0x7f + number_integer | 128..255 | uint 8 | 0xcc + number_integer | 256..65535 | uint 16 | 0xcd + number_integer | 65536..4294967295 | uint 32 | 0xce + number_integer | 4294967296..18446744073709551615 | uint 64 | 0xcf + number_unsigned | 0..127 | positive fixint | 0x00..0x7f + number_unsigned | 128..255 | uint 8 | 0xcc + number_unsigned | 256..65535 | uint 16 | 0xcd + number_unsigned | 65536..4294967295 | uint 32 | 0xce + number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xcf + number_float | *any value* | float 64 | 0xcb + string | *length*: 0..31 | fixstr | 0xa0..0xbf + string | *length*: 32..255 | str 8 | 0xd9 + string | *length*: 256..65535 | str 16 | 0xda + string | *length*: 65536..4294967295 | str 32 | 0xdb + array | *size*: 0..15 | fixarray | 0x90..0x9f + array | *size*: 16..65535 | array 16 | 0xdc + array | *size*: 65536..4294967295 | array 32 | 0xdd + object | *size*: 0..15 | fix map | 0x80..0x8f + object | *size*: 16..65535 | map 16 | 0xde + object | *size*: 65536..4294967295 | map 32 | 0xdf + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a MessagePack value. + + @note The following values can **not** be converted to a MessagePack value: + - strings with more than 4294967295 bytes + - arrays with more than 4294967295 elements + - objects with more than 4294967295 elements + + @note The following MessagePack types are not used in the conversion: + - bin 8 - bin 32 (0xc4..0xc6) + - ext 8 - ext 32 (0xc7..0xc9) + - float 32 (0xca) + - fixext 1 - fixext 16 (0xd4..0xd8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -7630,9 +9087,11 @@ class basic_json vector in MessagePack format.,to_msgpack} @sa http://msgpack.org - @sa @ref from_msgpack(const std::vector&) for the analogous - deserialization + @sa @ref from_msgpack(const std::vector&, const size_t) for the + analogous deserialization @sa @ref to_cbor(const basic_json& for the related CBOR format + + @since version 2.0.9 */ static std::vector to_msgpack(const basic_json& j) { @@ -7647,12 +9106,54 @@ class basic_json Deserializes a given byte vector @a v to a JSON value using the MessagePack serialization format. + The library maps MessagePack types to JSON value types as follows: + + MessagePack type | JSON value type | first byte + ---------------- | --------------- | ---------- + positive fixint | number_unsigned | 0x00..0x7f + fixmap | object | 0x80..0x8f + fixarray | array | 0x90..0x9f + fixstr | string | 0xa0..0xbf + nil | `null` | 0xc0 + false | `false` | 0xc2 + true | `true` | 0xc3 + float 32 | number_float | 0xca + float 64 | number_float | 0xcb + uint 8 | number_unsigned | 0xcc + uint 16 | number_unsigned | 0xcd + uint 32 | number_unsigned | 0xce + uint 64 | number_unsigned | 0xcf + int 8 | number_integer | 0xd0 + int 16 | number_integer | 0xd1 + int 32 | number_integer | 0xd2 + int 64 | number_integer | 0xd3 + str 8 | string | 0xd9 + str 16 | string | 0xda + str 32 | string | 0xdb + array 16 | array | 0xdc + array 32 | array | 0xdd + map 16 | object | 0xde + map 32 | object | 0xdf + negative fixint | number_integer | 0xe0-0xff + + @warning The mapping is **incomplete** in the sense that not all + MessagePack types can be converted to a JSON value. The following + MessagePack types are not supported and will yield parse errors: + - bin 8 - bin 32 (0xc4..0xc6) + - ext 8 - ext 32 (0xc7..0xc9) + - fixext 1 - fixext 16 (0xd4..0xd8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + @param[in] v a byte vector in MessagePack format + @param[in] start_index the index to start reading from @a v (0 by default) @return deserialized JSON value - @throw std::invalid_argument if unsupported features from MessagePack were + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from MessagePack were used in the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.113 if a string was expected as map key, but not found @complexity Linear in the size of the byte vector @a v. @@ -7661,11 +9162,15 @@ class basic_json @sa http://msgpack.org @sa @ref to_msgpack(const basic_json&) for the analogous serialization - @sa @ref from_cbor(const std::vector&) for the related CBOR format + @sa @ref from_cbor(const std::vector&, const size_t) for the + related CBOR format + + @since version 2.0.9, parameter @a start_index since 2.1.1 */ - static basic_json from_msgpack(const std::vector& v) + static basic_json from_msgpack(const std::vector& v, + const size_t start_index = 0) { - size_t i = 0; + size_t i = start_index; return from_msgpack_internal(v, i); } @@ -7677,6 +9182,65 @@ class basic_json serialization format which aims to be more compact than JSON itself, yet more efficient to parse. + The library uses the following mapping from JSON values types to + CBOR types according to the CBOR specification (RFC 7049): + + JSON value type | value/range | CBOR type | first byte + --------------- | ------------------------------------------ | ---------------------------------- | --------------- + null | `null` | Null | 0xf6 + boolean | `true` | True | 0xf5 + boolean | `false` | False | 0xf4 + number_integer | -9223372036854775808..-2147483649 | Negative integer (8 bytes follow) | 0x3b + number_integer | -2147483648..-32769 | Negative integer (4 bytes follow) | 0x3a + number_integer | -32768..-129 | Negative integer (2 bytes follow) | 0x39 + number_integer | -128..-25 | Negative integer (1 byte follow) | 0x38 + number_integer | -24..-1 | Negative integer | 0x20..0x37 + number_integer | 0..23 | Integer | 0x00..0x17 + number_integer | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_integer | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_integer | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a + number_integer | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_unsigned | 0..23 | Integer | 0x00..0x17 + number_unsigned | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_unsigned | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_unsigned | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a + number_unsigned | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_float | *any value* | Double-Precision Float | 0xfb + string | *length*: 0..23 | UTF-8 string | 0x60..0x77 + string | *length*: 23..255 | UTF-8 string (1 byte follow) | 0x78 + string | *length*: 256..65535 | UTF-8 string (2 bytes follow) | 0x79 + string | *length*: 65536..4294967295 | UTF-8 string (4 bytes follow) | 0x7a + string | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow) | 0x7b + array | *size*: 0..23 | array | 0x80..0x97 + array | *size*: 23..255 | array (1 byte follow) | 0x98 + array | *size*: 256..65535 | array (2 bytes follow) | 0x99 + array | *size*: 65536..4294967295 | array (4 bytes follow) | 0x9a + array | *size*: 4294967296..18446744073709551615 | array (8 bytes follow) | 0x9b + object | *size*: 0..23 | map | 0xa0..0xb7 + object | *size*: 23..255 | map (1 byte follow) | 0xb8 + object | *size*: 256..65535 | map (2 bytes follow) | 0xb9 + object | *size*: 65536..4294967295 | map (4 bytes follow) | 0xba + object | *size*: 4294967296..18446744073709551615 | map (8 bytes follow) | 0xbb + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a CBOR value. + + @note The following CBOR types are not used in the conversion: + - byte strings (0x40..0x5f) + - UTF-8 strings terminated by "break" (0x7f) + - arrays terminated by "break" (0x9f) + - maps terminated by "break" (0xbf) + - date/time (0xc0..0xc1) + - bignum (0xc2..0xc3) + - decimal fraction (0xc4) + - bigfloat (0xc5) + - tagged items (0xc6..0xd4, 0xd8..0xdb) + - expected conversions (0xd5..0xd7) + - simple values (0xe0..0xf3, 0xf8) + - undefined (0xf7) + - half and single-precision floats (0xf9-0xfa) + - break (0xff) + @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -7686,9 +9250,11 @@ class basic_json vector in CBOR format.,to_cbor} @sa http://cbor.io - @sa @ref from_cbor(const std::vector&) for the analogous - deserialization + @sa @ref from_cbor(const std::vector&, const size_t) for the + analogous deserialization @sa @ref to_msgpack(const basic_json& for the related MessagePack format + + @since version 2.0.9 */ static std::vector to_cbor(const basic_json& j) { @@ -7703,12 +9269,74 @@ class basic_json Deserializes a given byte vector @a v to a JSON value using the CBOR (Concise Binary Object Representation) serialization format. + The library maps CBOR types to JSON value types as follows: + + CBOR type | JSON value type | first byte + ---------------------- | --------------- | ---------- + Integer | number_unsigned | 0x00..0x17 + Unsigned integer | number_unsigned | 0x18 + Unsigned integer | number_unsigned | 0x19 + Unsigned integer | number_unsigned | 0x1a + Unsigned integer | number_unsigned | 0x1b + Negative integer | number_integer | 0x20..0x37 + Negative integer | number_integer | 0x38 + Negative integer | number_integer | 0x39 + Negative integer | number_integer | 0x3a + Negative integer | number_integer | 0x3b + Negative integer | number_integer | 0x40..0x57 + UTF-8 string | string | 0x60..0x77 + UTF-8 string | string | 0x78 + UTF-8 string | string | 0x79 + UTF-8 string | string | 0x7a + UTF-8 string | string | 0x7b + UTF-8 string | string | 0x7f + array | array | 0x80..0x97 + array | array | 0x98 + array | array | 0x99 + array | array | 0x9a + array | array | 0x9b + array | array | 0x9f + map | object | 0xa0..0xb7 + map | object | 0xb8 + map | object | 0xb9 + map | object | 0xba + map | object | 0xbb + map | object | 0xbf + False | `false` | 0xf4 + True | `true` | 0xf5 + Nill | `null` | 0xf6 + Half-Precision Float | number_float | 0xf9 + Single-Precision Float | number_float | 0xfa + Double-Precision Float | number_float | 0xfb + + @warning The mapping is **incomplete** in the sense that not all CBOR + types can be converted to a JSON value. The following CBOR types + are not supported and will yield parse errors (parse_error.112): + - byte strings (0x40..0x5f) + - date/time (0xc0..0xc1) + - bignum (0xc2..0xc3) + - decimal fraction (0xc4) + - bigfloat (0xc5) + - tagged items (0xc6..0xd4, 0xd8..0xdb) + - expected conversions (0xd5..0xd7) + - simple values (0xe0..0xf3, 0xf8) + - undefined (0xf7) + + @warning CBOR allows map keys of any type, whereas JSON only allows + strings as keys in object values. Therefore, CBOR maps with keys + other than UTF-8 strings are rejected (parse_error.113). + + @note Any CBOR output created @ref to_cbor can be successfully parsed by + @ref from_cbor. + @param[in] v a byte vector in CBOR format + @param[in] start_index the index to start reading from @a v (0 by default) @return deserialized JSON value - @throw std::invalid_argument if unsupported features from CBOR were used in - the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from CBOR were + used in the given vector @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found @complexity Linear in the size of the byte vector @a v. @@ -7717,375 +9345,63 @@ class basic_json @sa http://cbor.io @sa @ref to_cbor(const basic_json&) for the analogous serialization - @sa @ref from_msgpack(const std::vector&) for the related - MessagePack format - */ - static basic_json from_cbor(const std::vector& v) - { - size_t i = 0; - return from_cbor_internal(v, i); - } - - /// @} - - private: - /////////////////////////// - // convenience functions // - /////////////////////////// - - /*! - @brief return the type as string - - Returns the type name as string to be used in error messages - usually to - indicate that a function was called on a wrong JSON type. - - @return basically a string representation of a the @a m_type member - - @complexity Constant. - - @since version 1.0.0 - */ - std::string type_name() const - { - switch (m_type) - { - case value_t::null: - return "null"; - case value_t::object: - return "object"; - case value_t::array: - return "array"; - case value_t::string: - return "string"; - case value_t::boolean: - return "boolean"; - case value_t::discarded: - return "discarded"; - default: - return "number"; - } - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s) noexcept - { - return std::accumulate(s.begin(), s.end(), size_t{}, - [](size_t res, typename string_t::value_type c) - { - switch (c) - { - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - return res + 1; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // from c (1 byte) to \uxxxx (6 bytes) - return res + 5; - } - else - { - return res; - } - } - } - }); - } - - /*! - @brief escape a string - - Escape a string by replacing certain special characters by a sequence of - an escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. - - @param[in] s the string to escape - @return the escaped string - - @complexity Linear in the length of string @a s. - */ - static string_t escape_string(const string_t& s) - { - const auto space = extra_space(s); - if (space == 0) - { - return s; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; - - for (const auto& c : s) - { - switch (c) - { - // quotation mark (0x22) - case '"': - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - // reverse solidus (0x5c) - case '\\': - { - // nothing to change - pos += 2; - break; - } - - // backspace (0x08) - case '\b': - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - // formfeed (0x0c) - case '\f': - { - result[pos + 1] = 'f'; - pos += 2; - break; - } - - // newline (0x0a) - case '\n': - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - // carriage return (0x0d) - case '\r': - { - result[pos + 1] = 'r'; - pos += 2; - break; - } - - // horizontal tab (0x09) - case '\t': - { - result[pos + 1] = 't'; - pos += 2; - break; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // convert a number 0..15 to its hex representation - // (0..f) - static const char hexify[16] = - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - - // print character c as \uxxxx - for (const char m : - { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] - }) - { - result[++pos] = m; - } - - ++pos; - } - else - { - // all other characters are added as-is - result[pos++] = c; - } - break; - } - } - } - - return result; - } - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. Note that - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[out] o stream to write to - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(std::ostream& o, - const bool pretty_print, - const unsigned int indent_step, - const unsigned int current_indent = 0) const - { - // variable to hold indentation for recursive calls - unsigned int new_indent = current_indent; - - switch (m_type) - { - case value_t::object: - { - if (m_value.object->empty()) - { - o << "{}"; - return; - } - - o << "{"; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) - { - if (i != m_value.object->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' ') << "\"" - << escape_string(i->first) << "\":" - << (pretty_print ? " " : ""); - i->second.dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') + "}"; - return; - } - - case value_t::array: - { - if (m_value.array->empty()) - { - o << "[]"; - return; - } - - o << "["; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) - { - if (i != m_value.array->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' '); - i->dump(o, pretty_print, indent_step, new_indent); - } + @sa @ref from_msgpack(const std::vector&, const size_t) for the + related MessagePack format - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } + @since version 2.0.9, parameter @a start_index since 2.1.1 + */ + static basic_json from_cbor(const std::vector& v, + const size_t start_index = 0) + { + size_t i = start_index; + return from_cbor_internal(v, i); + } - o << string_t(new_indent, ' ') << "]"; - return; - } + /// @} - case value_t::string: - { - o << string_t("\"") << escape_string(*m_value.string) << "\""; - return; - } + /////////////////////////// + // convenience functions // + /////////////////////////// - case value_t::boolean: - { - o << (m_value.boolean ? "true" : "false"); - return; - } + /*! + @brief return the type as string - case value_t::number_integer: - { - o << m_value.number_integer; - return; - } + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. - case value_t::number_unsigned: - { - o << m_value.number_unsigned; - return; - } + @return basically a string representation of a the @a m_type member - case value_t::number_float: - { - if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); - } - else - { - o << m_value.number_float; - } - return; - } + @complexity Constant. - case value_t::discarded: - { - o << ""; - return; - } + @liveexample{The following code exemplifies `type_name()` for all JSON + types.,type_name} - case value_t::null: + @since version 1.0.0, public since 2.1.0 + */ + std::string type_name() const + { + { + switch (m_type) { - o << "null"; - return; + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; } } } + private: ////////////////////// // member variables // @@ -8115,6 +9431,11 @@ class basic_json class primitive_iterator_t { public: + + difference_type get_value() const noexcept + { + return m_it; + } /// set iterator to a defined beginning void set_begin() noexcept { @@ -8139,16 +9460,89 @@ class basic_json return (m_it == end_value); } - /// return reference to the value to change and compare - operator difference_type& () noexcept + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept { - return m_it; + return lhs.m_it == rhs.m_it; } - /// return value to compare - constexpr operator difference_type () const noexcept + friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept { - return m_it; + return !(lhs == rhs); + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it <= rhs.m_it; + } + + friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it > rhs.m_it; + } + + friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it >= rhs.m_it; + } + + primitive_iterator_t operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) + { + return os << it.m_it; + } + + primitive_iterator_t& operator++() + { + ++m_it; + return *this; + } + + primitive_iterator_t operator++(int) + { + auto result = *this; + m_it++; + return result; + } + + primitive_iterator_t& operator--() + { + --m_it; + return *this; + } + + primitive_iterator_t operator--(int) + { + auto result = *this; + m_it--; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) + { + m_it -= n; + return *this; } private: @@ -8500,7 +9894,7 @@ class basic_json case basic_json::value_t::null: { - throw std::out_of_range("cannot get value"); + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } default: @@ -8509,10 +9903,8 @@ class basic_json { return *m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8545,10 +9937,8 @@ class basic_json { return m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8648,7 +10038,7 @@ class basic_json // if objects are not the same, the comparison is undefined if (m_object != other.m_object) { - throw std::domain_error("cannot compare iterators of different containers"); + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); } assert(m_object != nullptr); @@ -8690,7 +10080,7 @@ class basic_json // if objects are not the same, the comparison is undefined if (m_object != other.m_object) { - throw std::domain_error("cannot compare iterators of different containers"); + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); } assert(m_object != nullptr); @@ -8699,7 +10089,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot compare order of object iterators"); + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators")); } case basic_json::value_t::array: @@ -8753,7 +10143,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use offsets with object iterators"); + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); } case basic_json::value_t::array: @@ -8815,7 +10205,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use offsets with object iterators"); + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); } case basic_json::value_t::array: @@ -8842,7 +10232,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use operator[] for object iterators"); + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators")); } case basic_json::value_t::array: @@ -8852,19 +10242,17 @@ class basic_json case basic_json::value_t::null: { - throw std::out_of_range("cannot get value"); + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } default: { - if (m_it.primitive_iterator == -n) + if (m_it.primitive_iterator.get_value() == -n) { return *m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8881,10 +10269,8 @@ class basic_json { return m_it.object_iterator->first; } - else - { - throw std::domain_error("cannot use key() for non-object iterators"); - } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators")); } /*! @@ -9039,7 +10425,9 @@ class basic_json literal_false, ///< the `false` literal literal_null, ///< the `null` literal value_string, ///< a string -- use get_string() for actual value - value_number, ///< a number -- use get_number() for actual value + value_unsigned, ///< an unsigned integer -- use get_number() for actual value + value_integer, ///< a signed integer -- use get_number() for actual value + value_float, ///< an floating point number -- use get_number() for actual value begin_array, ///< the character for array begin `[` begin_object, ///< the character for object begin `{` end_array, ///< the character for array end `]` @@ -9062,14 +10450,17 @@ class basic_json m_limit = m_content + len; } - /// a lexer from an input stream + /*! + @brief a lexer from an input stream + @throw parse_error.111 if input stream is in a bad state + */ explicit lexer(std::istream& s) : m_stream(&s), m_line_buffer() { // immediately abort if stream is erroneous if (s.fail()) { - throw std::invalid_argument("stream error"); + JSON_THROW(parse_error::create(111, 0, "bad input stream")); } // fill buffer @@ -9103,17 +10494,17 @@ class basic_json @return string representation of the code point; the length of the result string is between 1 and 4 characters. - @throw std::out_of_range if code point is > 0x10ffff; example: `"code - points above 0x10FFFF are invalid"` - @throw std::invalid_argument if the low surrogate is invalid; example: + @throw parse_error.102 if the low surrogate is invalid; example: `""missing or wrong low surrogate""` + @throw parse_error.103 if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` @complexity Constant. @see */ - static string_t to_unicode(const std::size_t codepoint1, - const std::size_t codepoint2 = 0) + string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) const { // calculate the code point from the given code points std::size_t codepoint = codepoint1; @@ -9136,7 +10527,7 @@ class basic_json } else { - throw std::invalid_argument("missing or wrong low surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing or wrong low surrogate")); } } @@ -9150,27 +10541,27 @@ class basic_json else if (codepoint <= 0x7ff) { // 2-byte characters: 110xxxxx 10xxxxxx - result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0xC0 | (codepoint >> 6))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0xffff) { // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0xE0 | (codepoint >> 12))); result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0x10ffff) { // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0xF0 | (codepoint >> 18))); result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else { - throw std::out_of_range("code points above 0x10FFFF are invalid"); + JSON_THROW(parse_error::create(103, get_position(), "code points above 0x10FFFF are invalid")); } return result; @@ -9191,7 +10582,9 @@ class basic_json return "null literal"; case token_type::value_string: return "string literal"; - case token_type::value_number: + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: return "number literal"; case token_type::begin_array: return "'['"; @@ -9233,10 +10626,9 @@ class basic_json Proof (by contradiction): Assume a finite input. To loop forever, the loop must never hit code with a `break` statement. The only code - snippets without a `break` statement are the continue statements for - whitespace and byte-order-marks. To loop forever, the input must be an - infinite sequence of whitespace or byte-order-marks. This contradicts - the assumption of finite input, q.e.d. + snippets without a `break` statement is the continue statement for + whitespace. To loop forever, the input must be an infinite sequence + whitespace. This contradicts the assumption of finite input, q.e.d. */ token_type scan() { @@ -9428,6 +10820,7 @@ basic_json_parser_6: goto basic_json_parser_6; } { + position += static_cast((m_cursor - m_start)); continue; } basic_json_parser_9: @@ -9464,37 +10857,47 @@ basic_json_parser_12: } if (yych <= '0') { - goto basic_json_parser_13; + goto basic_json_parser_43; } if (yych <= '9') { - goto basic_json_parser_15; + goto basic_json_parser_45; } goto basic_json_parser_5; basic_json_parser_13: yyaccept = 1; yych = *(m_marker = ++m_cursor); - if (yych <= 'D') + if (yych <= '9') { if (yych == '.') { - goto basic_json_parser_43; + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; } } else { if (yych <= 'E') { - goto basic_json_parser_44; + if (yych >= 'E') + { + goto basic_json_parser_51; + } } - if (yych == 'e') + else { - goto basic_json_parser_44; + if (yych == 'e') + { + goto basic_json_parser_51; + } } } basic_json_parser_14: { - last_token_type = token_type::value_number; + last_token_type = token_type::value_unsigned; break; } basic_json_parser_15: @@ -9513,7 +10916,7 @@ basic_json_parser_15: { if (yych == '.') { - goto basic_json_parser_43; + goto basic_json_parser_47; } goto basic_json_parser_14; } @@ -9521,11 +10924,11 @@ basic_json_parser_15: { if (yych <= 'E') { - goto basic_json_parser_44; + goto basic_json_parser_51; } if (yych == 'e') { - goto basic_json_parser_44; + goto basic_json_parser_51; } goto basic_json_parser_14; } @@ -9552,7 +10955,7 @@ basic_json_parser_23: yych = *(m_marker = ++m_cursor); if (yych == 'a') { - goto basic_json_parser_45; + goto basic_json_parser_52; } goto basic_json_parser_5; basic_json_parser_24: @@ -9560,7 +10963,7 @@ basic_json_parser_24: yych = *(m_marker = ++m_cursor); if (yych == 'u') { - goto basic_json_parser_46; + goto basic_json_parser_53; } goto basic_json_parser_5; basic_json_parser_25: @@ -9568,7 +10971,7 @@ basic_json_parser_25: yych = *(m_marker = ++m_cursor); if (yych == 'r') { - goto basic_json_parser_47; + goto basic_json_parser_54; } goto basic_json_parser_5; basic_json_parser_26: @@ -9650,13 +11053,27 @@ basic_json_parser_31: } basic_json_parser_32: m_cursor = m_marker; - if (yyaccept == 0) + if (yyaccept <= 1) { - goto basic_json_parser_5; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } } else { - goto basic_json_parser_14; + if (yyaccept == 2) + { + goto basic_json_parser_44; + } + else + { + goto basic_json_parser_58; + } } basic_json_parser_33: ++m_cursor; @@ -9737,7 +11154,7 @@ basic_json_parser_35: } if (yych <= 'u') { - goto basic_json_parser_48; + goto basic_json_parser_55; } goto basic_json_parser_32; } @@ -9836,43 +11253,138 @@ basic_json_parser_41: } if (yych <= 0xBF) { - goto basic_json_parser_38; + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_42: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0x8F) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_43: + yyaccept = 2; + yych = *(m_marker = ++m_cursor); + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; + } + } + else + { + if (yych <= 'E') + { + if (yych >= 'E') + { + goto basic_json_parser_51; + } + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + } + } +basic_json_parser_44: + { + last_token_type = token_type::value_integer; + break; + } +basic_json_parser_45: + yyaccept = 2; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych <= '/') + { + goto basic_json_parser_44; + } + goto basic_json_parser_45; + } + else + { + if (yych <= 'E') + { + if (yych <= 'D') + { + goto basic_json_parser_44; + } + goto basic_json_parser_51; + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + goto basic_json_parser_44; + } + } +basic_json_parser_47: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_56; } goto basic_json_parser_32; -basic_json_parser_42: +basic_json_parser_48: ++m_cursor; if (m_limit <= m_cursor) { fill_line_buffer(1); // LCOV_EXCL_LINE } yych = *m_cursor; - if (yych <= 0x7F) - { - goto basic_json_parser_32; - } - if (yych <= 0x8F) - { - goto basic_json_parser_38; - } - goto basic_json_parser_32; -basic_json_parser_43: - yych = *++m_cursor; if (yych <= '/') { - goto basic_json_parser_32; + goto basic_json_parser_50; } if (yych <= '9') { - goto basic_json_parser_49; + goto basic_json_parser_48; } - goto basic_json_parser_32; -basic_json_parser_44: +basic_json_parser_50: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_51: yych = *++m_cursor; if (yych <= ',') { if (yych == '+') { - goto basic_json_parser_51; + goto basic_json_parser_59; } goto basic_json_parser_32; } @@ -9880,7 +11392,7 @@ basic_json_parser_44: { if (yych <= '-') { - goto basic_json_parser_51; + goto basic_json_parser_59; } if (yych <= '/') { @@ -9888,32 +11400,32 @@ basic_json_parser_44: } if (yych <= '9') { - goto basic_json_parser_52; + goto basic_json_parser_60; } goto basic_json_parser_32; } -basic_json_parser_45: +basic_json_parser_52: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_54; + goto basic_json_parser_62; } goto basic_json_parser_32; -basic_json_parser_46: +basic_json_parser_53: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_55; + goto basic_json_parser_63; } goto basic_json_parser_32; -basic_json_parser_47: +basic_json_parser_54: yych = *++m_cursor; if (yych == 'u') { - goto basic_json_parser_56; + goto basic_json_parser_64; } goto basic_json_parser_32; -basic_json_parser_48: +basic_json_parser_55: ++m_cursor; if (m_limit <= m_cursor) { @@ -9928,7 +11440,7 @@ basic_json_parser_48: } if (yych <= '9') { - goto basic_json_parser_57; + goto basic_json_parser_65; } goto basic_json_parser_32; } @@ -9936,7 +11448,7 @@ basic_json_parser_48: { if (yych <= 'F') { - goto basic_json_parser_57; + goto basic_json_parser_65; } if (yych <= '`') { @@ -9944,12 +11456,12 @@ basic_json_parser_48: } if (yych <= 'f') { - goto basic_json_parser_57; + goto basic_json_parser_65; } goto basic_json_parser_32; } -basic_json_parser_49: - yyaccept = 1; +basic_json_parser_56: + yyaccept = 3; m_marker = ++m_cursor; if ((m_limit - m_cursor) < 3) { @@ -9960,27 +11472,30 @@ basic_json_parser_49: { if (yych <= '/') { - goto basic_json_parser_14; + goto basic_json_parser_58; } if (yych <= '9') { - goto basic_json_parser_49; + goto basic_json_parser_56; } - goto basic_json_parser_14; } else { if (yych <= 'E') { - goto basic_json_parser_44; + goto basic_json_parser_51; } if (yych == 'e') { - goto basic_json_parser_44; + goto basic_json_parser_51; } - goto basic_json_parser_14; } -basic_json_parser_51: +basic_json_parser_58: + { + last_token_type = token_type::value_float; + break; + } +basic_json_parser_59: yych = *++m_cursor; if (yych <= '/') { @@ -9990,7 +11505,7 @@ basic_json_parser_51: { goto basic_json_parser_32; } -basic_json_parser_52: +basic_json_parser_60: ++m_cursor; if (m_limit <= m_cursor) { @@ -9999,35 +11514,35 @@ basic_json_parser_52: yych = *m_cursor; if (yych <= '/') { - goto basic_json_parser_14; + goto basic_json_parser_58; } if (yych <= '9') { - goto basic_json_parser_52; + goto basic_json_parser_60; } - goto basic_json_parser_14; -basic_json_parser_54: + goto basic_json_parser_58; +basic_json_parser_62: yych = *++m_cursor; if (yych == 's') { - goto basic_json_parser_58; + goto basic_json_parser_66; } goto basic_json_parser_32; -basic_json_parser_55: +basic_json_parser_63: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_59; + goto basic_json_parser_67; } goto basic_json_parser_32; -basic_json_parser_56: +basic_json_parser_64: yych = *++m_cursor; if (yych == 'e') { - goto basic_json_parser_61; + goto basic_json_parser_69; } goto basic_json_parser_32; -basic_json_parser_57: +basic_json_parser_65: ++m_cursor; if (m_limit <= m_cursor) { @@ -10042,7 +11557,7 @@ basic_json_parser_57: } if (yych <= '9') { - goto basic_json_parser_63; + goto basic_json_parser_71; } goto basic_json_parser_32; } @@ -10050,7 +11565,7 @@ basic_json_parser_57: { if (yych <= 'F') { - goto basic_json_parser_63; + goto basic_json_parser_71; } if (yych <= '`') { @@ -10058,30 +11573,30 @@ basic_json_parser_57: } if (yych <= 'f') { - goto basic_json_parser_63; + goto basic_json_parser_71; } goto basic_json_parser_32; } -basic_json_parser_58: +basic_json_parser_66: yych = *++m_cursor; if (yych == 'e') { - goto basic_json_parser_64; + goto basic_json_parser_72; } goto basic_json_parser_32; -basic_json_parser_59: +basic_json_parser_67: ++m_cursor; { last_token_type = token_type::literal_null; break; } -basic_json_parser_61: +basic_json_parser_69: ++m_cursor; { last_token_type = token_type::literal_true; break; } -basic_json_parser_63: +basic_json_parser_71: ++m_cursor; if (m_limit <= m_cursor) { @@ -10096,7 +11611,7 @@ basic_json_parser_63: } if (yych <= '9') { - goto basic_json_parser_66; + goto basic_json_parser_74; } goto basic_json_parser_32; } @@ -10104,7 +11619,7 @@ basic_json_parser_63: { if (yych <= 'F') { - goto basic_json_parser_66; + goto basic_json_parser_74; } if (yych <= '`') { @@ -10112,17 +11627,17 @@ basic_json_parser_63: } if (yych <= 'f') { - goto basic_json_parser_66; + goto basic_json_parser_74; } goto basic_json_parser_32; } -basic_json_parser_64: +basic_json_parser_72: ++m_cursor; { last_token_type = token_type::literal_false; break; } -basic_json_parser_66: +basic_json_parser_74: ++m_cursor; if (m_limit <= m_cursor) { @@ -10161,6 +11676,7 @@ basic_json_parser_66: } + position += static_cast((m_cursor - m_start)); return last_token_type; } @@ -10209,7 +11725,7 @@ basic_json_parser_66: assert(m_marker == nullptr or m_marker <= m_limit); // number of processed characters (p) - const size_t num_processed_chars = static_cast(m_start - m_content); + const auto num_processed_chars = static_cast(m_start - m_content); // offset for m_marker wrt. to m_start const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; // number of unprocessed characters (u) @@ -10219,7 +11735,7 @@ basic_json_parser_66: if (m_stream == nullptr or m_stream->eof()) { // m_start may or may not be pointing into m_line_buffer at - // this point. We trust the standand library to do the right + // this point. We trust the standard library to do the right // thing. See http://stackoverflow.com/q/28142011/266378 m_line_buffer.assign(m_start, m_limit); @@ -10237,6 +11753,13 @@ basic_json_parser_66: m_line_buffer.erase(0, num_processed_chars); // read next line from input stream m_line_buffer_tmp.clear(); + + // check if stream is still good + if (m_stream->fail()) + { + JSON_THROW(parse_error::create(111, 0, "bad input stream")); + } + std::getline(*m_stream, m_line_buffer_tmp, '\n'); // add line with newline symbol to the line buffer @@ -10307,7 +11830,7 @@ basic_json_parser_66: m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This can be rephrased to m_cursor - m_start - 2 > x. With the precondition, we x <= 0, meaning that the loop condition holds - indefinitly if i is always decreased. However, observe that the value + indefinitely if i is always decreased. However, observe that the value of i is strictly increasing with each iteration, as it is incremented by 1 in the iteration expression and never decremented inside the loop body. Hence, the loop condition will eventually be false which @@ -10316,7 +11839,8 @@ basic_json_parser_66: @return string value of current token without opening and closing quotes - @throw std::out_of_range if to_unicode fails + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails */ string_t get_string() const { @@ -10402,7 +11926,7 @@ basic_json_parser_66: // make sure there is a subsequent unicode if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') { - throw std::invalid_argument("missing low surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing low surrogate")); } // get code yyyy from uxxxx\uyyyy @@ -10415,7 +11939,7 @@ basic_json_parser_66: else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF) { // we found a lone low surrogate - throw std::invalid_argument("missing high surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing high surrogate")); } else { @@ -10433,186 +11957,245 @@ basic_json_parser_66: return result; } - /*! - @brief parse floating point number - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + /*! + @brief parse string into a built-in arithmetic type as if the current + locale is POSIX. - @param[in,out] endptr recieves a pointer to the first character after - the number + @note in floating-point case strtod may parse past the token's end - + this is not an error - @return the floating point number + @note any leading blanks are not handled */ - long double str_to_float_t(long double* /* type */, char** endptr) const + struct strtonum { - return std::strtold(reinterpret_cast(m_start), endptr); - } + public: + strtonum(const char* start, const char* end) + : m_start(start), m_end(end) + {} - /*! - @brief parse floating point number + /*! + @return true iff parsed successfully as number of type T - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + @param[in,out] val shall contain parsed value, or undefined value + if could not parse + */ + template::value>::type> + bool to(T& val) const + { + return parse(val, std::is_integral()); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + private: + const char* const m_start = nullptr; + const char* const m_end = nullptr; - @return the floating point number - */ - double str_to_float_t(double* /* type */, char** endptr) const - { - return std::strtod(reinterpret_cast(m_start), endptr); - } + // floating-point conversion - /*! - @brief parse floating point number + // overloaded wrappers for strtod/strtof/strtold + // that will be called from parse + static void strtof(float& f, const char* str, char** endptr) + { + f = std::strtof(str, endptr); + } - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + static void strtof(double& f, const char* str, char** endptr) + { + f = std::strtod(str, endptr); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + static void strtof(long double& f, const char* str, char** endptr) + { + f = std::strtold(str, endptr); + } - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } + template + bool parse(T& value, /*is_integral=*/std::false_type) const + { + // replace decimal separator with locale-specific version, + // when necessary; data will point to either the original + // string, or buf, or tempstr containing the fixed string. + std::string tempstr; + std::array buf; + const size_t len = static_cast(m_end - m_start); - /*! - @brief return number value for number tokens + // lexer will reject empty numbers + assert(len > 0); - This function translates the last token into the most appropriate - number type (either integer, unsigned integer or floating point), - which is passed back to the caller via the result parameter. + // since dealing with strtod family of functions, we're + // getting the decimal point char from the C locale facilities + // instead of C++'s numpunct facet of the current std::locale + const auto loc = localeconv(); + assert(loc != nullptr); + const char decimal_point_char = (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0]; - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. + const char* data = m_start; - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). + if (decimal_point_char != '.') + { + const size_t ds_pos = static_cast(std::find(m_start, m_end, '.') - m_start); - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. - */ - void get_number(basic_json& result) const - { - assert(m_start != nullptr); + if (ds_pos != len) + { + // copy the data into the local buffer or tempstr, if + // buffer is too small; replace decimal separator, and + // update data to point to the modified bytes + if ((len + 1) < buf.size()) + { + std::copy(m_start, m_end, buf.begin()); + buf[len] = 0; + buf[ds_pos] = decimal_point_char; + data = buf.data(); + } + else + { + tempstr.assign(m_start, m_end); + tempstr[ds_pos] = decimal_point_char; + data = tempstr.c_str(); + } + } + } + + char* endptr = nullptr; + value = 0; + // this calls appropriate overload depending on T + strtof(value, data, &endptr); - const lexer::lexer_char_t* curptr = m_start; + // parsing was successful iff strtof parsed exactly the number + // of characters determined by the lexer (len) + const bool ok = (endptr == (data + len)); - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; + if (ok and (value == static_cast(0.0)) and (*data == '-')) + { + // some implementations forget to negate the zero + value = -0.0; + } - // maximum absolute value of the relevant integer type - number_unsigned_t max; + return ok; + } - // temporarily store the type to avoid unecessary bitfield access - value_t type; + // integral conversion - // look for sign - if (*curptr == '-') + signed long long parse_integral(char** endptr, /*is_signed*/std::true_type) const { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; + return std::strtoll(m_start, endptr, 10); } - else + + unsigned long long parse_integral(char** endptr, /*is_signed*/std::false_type) const + { + return std::strtoull(m_start, endptr, 10); + } + + template + bool parse(T& value, /*is_integral=*/std::true_type) const { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); + char* endptr = nullptr; + errno = 0; // these are thread-local + const auto x = parse_integral(&endptr, std::is_signed()); + + // called right overload? + static_assert(std::is_signed() == std::is_signed(), ""); + + value = static_cast(x); + + return (x == static_cast(value)) // x fits into destination T + and (x < 0) == (value < 0) // preserved sign + //and ((x != 0) or is_integral()) // strto[u]ll did nto fail + and (errno == 0) // strto[u]ll did not overflow + and (m_start < m_end) // token was not empty + and (endptr == m_end); // parsed entire token exactly } + }; + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + integral numbers that don't fit into the the range of the respective + type are parsed as number_float_t + + floating-point values do not satisfy std::isfinite predicate + are converted to value_t::null + + throws if the entire string [m_start .. m_cursor) cannot be + interpreted as a number + + @param[out] result @ref basic_json object to receive the number. + @param[in] token the type of the number token + */ + bool get_number(basic_json& result, const token_type token) const + { + assert(m_start != nullptr); + assert(m_start < m_cursor); + assert((token == token_type::value_unsigned) or + (token == token_type::value_integer) or + (token == token_type::value_float)); - // count the significant figures - for (; curptr < m_cursor; curptr++) + strtonum num_converter(reinterpret_cast(m_start), + reinterpret_cast(m_cursor)); + + switch (token) { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') + case lexer::token_type::value_unsigned: { - if (*curptr == '.') + number_unsigned_t val; + if (num_converter.to(val)) { - // don't count '.' but change to float - type = value_t::number_float; - continue; + // parsing successful + result.m_type = value_t::number_unsigned; + result.m_value = val; + return true; } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; break; } - // skip if definitely not an integer - if (type != value_t::number_float) + case lexer::token_type::value_integer: { - auto digit = static_cast(*curptr - '0'); - - // overflow if value * 10 + digit > max, move terms around - // to avoid overflow in intermediate values - if (value > (max - digit) / 10) - { - // overflow - type = value_t::number_float; - } - else + number_integer_t val; + if (num_converter.to(val)) { - // no overflow - value = value * 10 + digit; + // parsing successful + result.m_type = value_t::number_integer; + result.m_value = val; + return true; } + break; } - } - - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - // invariant: if we parsed a '-', the absolute value is between - // 0 (we allow -0) and max == -INT64_MIN - assert(value >= 0); - assert(value <= max); - if (value == max) - { - // we cannot simply negate value (== max == -INT64_MIN), - // see https://github.com/nlohmann/json/issues/389 - result.m_value.number_integer = static_cast(INT64_MIN); - } - else + default: { - // all other values can be negated safely - result.m_value.number_integer = -static_cast(value); + break; } } - else + + // parse float (either explicitly or because a previous conversion + // failed) + number_float_t val; + if (num_converter.to(val)) { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); + // parsing successful + result.m_type = value_t::number_float; + result.m_value = val; - // replace infinity and NAN by null + // throw in case of infinity or NAN if (not std::isfinite(result.m_value.number_float)) { - type = value_t::null; - result.m_value = basic_json::json_value(); + JSON_THROW(out_of_range::create(406, "number overflow parsing '" + get_token_string() + "'")); } + + return true; } - // save the type - result.m_type = type; + // couldn't parse number in any format + return false; + } + + constexpr size_t get_position() const + { + return position; } private: @@ -10634,6 +12217,8 @@ basic_json_parser_66: const lexer_char_t* m_limit = nullptr; /// the last token type token_type last_token_type = token_type::end_of_input; + /// current position in the input (read bytes) + size_t position = 0; }; /*! @@ -10650,7 +12235,10 @@ basic_json_parser_66: m_lexer(reinterpret_cast(buff), std::strlen(buff)) {} - /// a parser reading from an input stream + /*! + @brief a parser reading from an input stream + @throw parse_error.111 if input stream is in a bad state + */ parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) {} @@ -10666,7 +12254,12 @@ basic_json_parser_66: static_cast(std::distance(first, last))) {} - /// public parser interface + /*! + @brief public parser interface + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ basic_json parse() { // read first token @@ -10683,7 +12276,12 @@ basic_json_parser_66: } private: - /// the actual parser + /*! + @brief the actual parser + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ basic_json parse_internal(bool keep) { auto result = basic_json(value_t::discarded); @@ -10856,9 +12454,11 @@ basic_json_parser_66: break; } - case lexer::token_type::value_number: + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: { - m_lexer.get_number(result); + m_lexer.get_number(result, last_token); get_token(); break; } @@ -10884,6 +12484,9 @@ basic_json_parser_66: return last_token; } + /*! + @throw parse_error.101 if expected token did not occur + */ void expect(typename lexer::token_type t) const { if (t != last_token) @@ -10893,10 +12496,13 @@ basic_json_parser_66: "'") : lexer::token_type_name(last_token)); error_msg += "; expected " + lexer::token_type_name(t); - throw std::invalid_argument(error_msg); + JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); } } + /*! + @throw parse_error.101 if unexpected token occurred + */ void unexpect(typename lexer::token_type t) const { if (t == last_token) @@ -10905,7 +12511,7 @@ basic_json_parser_66: error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + "'") : lexer::token_type_name(last_token)); - throw std::invalid_argument(error_msg); + JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); } } @@ -10948,12 +12554,12 @@ basic_json_parser_66: empty string is assumed which references the whole JSON value - @throw std::domain_error if reference token is nonempty and does not - begin with a slash (`/`); example: `"JSON pointer must be empty or - begin with /"` - @throw std::domain_error if a tilde (`~`) is not followed by `0` - (representing `~`) or `1` (representing `/`); example: `"escape error: - ~ must be followed with 0 or 1"` + @throw parse_error.107 if the given JSON pointer @a s is nonempty and + does not begin with a slash (`/`); see example below + + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s + is not followed by `0` (representing `~`) or `1` (representing `/`); + see example below @liveexample{The example shows the construction several valid JSON pointers as well as the exceptional behavior.,json_pointer} @@ -10996,12 +12602,15 @@ basic_json_parser_66: } private: - /// remove and return last reference pointer + /*! + @brief remove and return last reference pointer + @throw out_of_range.405 if JSON pointer has no parent + */ std::string pop_back() { if (is_root()) { - throw std::domain_error("JSON pointer has no parent"); + JSON_THROW(out_of_range::create(405, "JSON pointer has no parent")); } auto last = reference_tokens.back(); @@ -11019,7 +12628,7 @@ basic_json_parser_66: { if (is_root()) { - throw std::domain_error("JSON pointer has no parent"); + JSON_THROW(out_of_range::create(405, "JSON pointer has no parent")); } json_pointer result = *this; @@ -11031,6 +12640,9 @@ basic_json_parser_66: @brief create and return a reference to the pointed to value @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened */ reference get_and_create(reference j) const { @@ -11067,7 +12679,14 @@ basic_json_parser_66: case value_t::array: { // create an entry in the array - result = &result->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + result = &result->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } @@ -11080,7 +12699,7 @@ basic_json_parser_66: */ default: { - throw std::domain_error("invalid value to unflatten"); + JSON_THROW(type_error::create(313, "invalid value to unflatten")); } } } @@ -11103,9 +12722,9 @@ basic_json_parser_66: @complexity Linear in the length of the JSON pointer. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved */ reference get_unchecked(pointer ptr) const { @@ -11119,7 +12738,7 @@ basic_json_parser_66: reference_token.end(), [](const char x) { - return std::isdigit(x); + return (x >= '0' and x <= '9'); }); // change value to array for numbers or "-" or to object @@ -11148,25 +12767,32 @@ basic_json_parser_66: // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } if (reference_token == "-") { - // explicityly treat "-" as index beyond the end + // explicitly treat "-" as index beyond the end ptr = &ptr->operator[](ptr->m_value.array->size()); } else { // convert array index to number; unchecked access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11174,6 +12800,12 @@ basic_json_parser_66: return *ptr; } + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ reference get_checked(pointer ptr) const { for (const auto& reference_token : reference_tokens) @@ -11192,25 +12824,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11225,6 +12864,11 @@ basic_json_parser_66: @return const reference to the JSON value pointed to by the JSON pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved */ const_reference get_unchecked(const_pointer ptr) const { @@ -11244,25 +12888,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" cannot be used for const access - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // use unchecked array access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11270,6 +12921,12 @@ basic_json_parser_66: return *ptr; } + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ const_reference get_checked(const_pointer ptr) const { for (const auto& reference_token : reference_tokens) @@ -11288,25 +12945,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11314,7 +12978,15 @@ basic_json_parser_66: return *ptr; } - /// split the string input to reference tokens + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ static std::vector split(const std::string& reference_string) { std::vector result; @@ -11328,7 +13000,7 @@ basic_json_parser_66: // check if nonempty reference string begins with slash if (reference_string[0] != '/') { - throw std::domain_error("JSON pointer must be empty or begin with '/'"); + JSON_THROW(parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'")); } // extract the reference tokens: @@ -11336,7 +13008,7 @@ basic_json_parser_66: // - start: position after the previous slash for ( // search for the first slash after the first character - size_t slash = reference_string.find_first_of("/", 1), + size_t slash = reference_string.find_first_of('/', 1), // set the beginning of the first reference token start = 1; // we can stop if start == string::npos+1 = 0 @@ -11345,16 +13017,16 @@ basic_json_parser_66: // (will eventually be 0 if slash == std::string::npos) start = slash + 1, // find next slash - slash = reference_string.find_first_of("/", start)) + slash = reference_string.find_first_of('/', start)) { // use the text between the beginning of the reference token // (start) and the last slash (slash). auto reference_token = reference_string.substr(start, slash - start); // check reference tokens are properly escaped - for (size_t pos = reference_token.find_first_of("~"); + for (size_t pos = reference_token.find_first_of('~'); pos != std::string::npos; - pos = reference_token.find_first_of("~", pos + 1)) + pos = reference_token.find_first_of('~', pos + 1)) { assert(reference_token[pos] == '~'); @@ -11363,7 +13035,7 @@ basic_json_parser_66: (reference_token[pos + 1] != '0' and reference_token[pos + 1] != '1')) { - throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); + JSON_THROW(parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'")); } } @@ -11375,7 +13047,6 @@ basic_json_parser_66: return result; } - private: /*! @brief replace all occurrences of a substring by another string @@ -11384,7 +13055,8 @@ basic_json_parser_66: @param[in] f the substring to replace with @a t @param[in] t the string to replace @a f - @pre The search string @a f must not be empty. + @pre The search string @a f must not be empty. **This precondition is + enforced with an assertion.** @since version 2.0.0 */ @@ -11484,12 +13156,17 @@ basic_json_parser_66: @param[in] value flattened JSON @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened */ static basic_json unflatten(const basic_json& value) { if (not value.is_object()) { - throw std::domain_error("only objects can be unflattened"); + JSON_THROW(type_error::create(314, "only objects can be unflattened")); } basic_json result; @@ -11499,7 +13176,7 @@ basic_json_parser_66: { if (not element.second.is_primitive()) { - throw std::domain_error("values in object must be primitive"); + JSON_THROW(type_error::create(315, "values in object must be primitive")); } // assign value to reference pointed to by JSON pointer; Note @@ -11513,7 +13190,18 @@ basic_json_parser_66: return result; } - private: + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + /// the reference tokens std::vector reference_tokens {}; }; @@ -11550,9 +13238,9 @@ basic_json_parser_66: @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved @liveexample{The behavior is shown in the example.,operatorjson_pointer} @@ -11577,9 +13265,10 @@ basic_json_parser_66: @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} @@ -11600,15 +13289,30 @@ basic_json_parser_66: @return reference to the element pointed to by @a ptr - @complexity Constant. + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. - @liveexample{The behavior is shown in the example.,at_json_pointer} + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. @since version 2.0.0 + + @liveexample{The behavior is shown in the example.,at_json_pointer} */ reference at(const json_pointer& ptr) { @@ -11625,15 +13329,30 @@ basic_json_parser_66: @return reference to the element pointed to by @a ptr - @complexity Constant. + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. - @liveexample{The behavior is shown in the example.,at_json_pointer_const} + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. @since version 2.0.0 + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} */ const_reference at(const json_pointer& ptr) const { @@ -11648,7 +13367,7 @@ basic_json_parser_66: primitive. The original JSON value can be restored using the @ref unflatten() function. - @return an object that maps JSON pointers to primitve values + @return an object that maps JSON pointers to primitive values @note Empty objects and arrays are flattened to `null` and will not be reconstructed correctly by the @ref unflatten() function. @@ -11689,6 +13408,9 @@ basic_json_parser_66: @complexity Linear in the size the JSON value. + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitve + @liveexample{The following code shows how a flattened JSON object is unflattened into the original nested JSON object.,unflatten} @@ -11715,7 +13437,7 @@ basic_json_parser_66: [JSON Patch](http://jsonpatch.com) defines a JSON document structure for expressing a sequence of operations to apply to a JSON) document. With - this funcion, a JSON Patch is applied to the current JSON value by + this function, a JSON Patch is applied to the current JSON value by executing all operations from the patch. @param[in] json_patch JSON patch document @@ -11726,12 +13448,23 @@ basic_json_parser_66: any case, the original value is not changed: the patch is applied to a copy of the value. - @throw std::out_of_range if a JSON pointer inside the patch could not - be resolved successfully in the current JSON value; example: `"key baz - not found"` - @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + @throw parse_error.104 if the JSON patch does not consist of an array of + objects + + @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory attributes are missing); example: `"operation add must have member path"` + @throw out_of_range.401 if an array index is out of range. + + @throw out_of_range.403 if a JSON pointer inside the patch could not be + resolved successfully in the current JSON value; example: `"key baz not + found"` + + @throw out_of_range.405 if JSON pointer has no parent ("add", "remove", + "move") + + @throw other_error.501 if "test" operation was unsuccessful + @complexity Linear in the size of the JSON value and the length of the JSON patch. As usually only a fraction of the JSON value is affected by the patch, the complexity can usually be neglected. @@ -11754,7 +13487,7 @@ basic_json_parser_66: // the valid JSON Patch operations enum class patch_operations {add, remove, replace, move, copy, test, invalid}; - const auto get_op = [](const std::string op) + const auto get_op = [](const std::string & op) { if (op == "add") { @@ -11828,7 +13561,7 @@ basic_json_parser_66: if (static_cast(idx) > parent.size()) { // avoid undefined behavior - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } else { @@ -11866,7 +13599,7 @@ basic_json_parser_66: } else { - throw std::out_of_range("key '" + last_path + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found")); } } else if (parent.is_array()) @@ -11876,14 +13609,13 @@ basic_json_parser_66: } }; - // type check + // type check: top level value must be an array if (not json_patch.is_array()) { - // a JSON patch must be an array of objects - throw std::invalid_argument("JSON patch must be an array of objects"); + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); } - // iterate and apply th eoperations + // iterate and apply the operations for (const auto& val : json_patch) { // wrapper to get a value for an operation @@ -11900,23 +13632,23 @@ basic_json_parser_66: // check if desired value is present if (it == val.m_value.object->end()) { - throw std::invalid_argument(error_msg + " must have member '" + member + "'"); + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'")); } // check if result is of type string if (string_type and not it->second.is_string()) { - throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'")); } // no error: return value return it->second; }; - // type check + // type check: every element of the array must be an object if (not val.is_object()) { - throw std::invalid_argument("JSON patch must be an array of objects"); + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); } // collect mandatory members @@ -11975,13 +13707,13 @@ basic_json_parser_66: case patch_operations::test: { bool success = false; - try + JSON_TRY { // check if "value" matches the one at "path" // the "path" location must exist - use at() success = (result.at(ptr) == get_value("test", "value", false)); } - catch (std::out_of_range&) + JSON_CATCH (out_of_range&) { // ignore out of range errors: success remains false } @@ -11989,7 +13721,7 @@ basic_json_parser_66: // throw an exception if test fails if (not success) { - throw std::domain_error("unsuccessful: " + val.dump()); + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump())); } break; @@ -11999,7 +13731,7 @@ basic_json_parser_66: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - throw std::invalid_argument("operation value '" + op + "' is invalid"); + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid")); } } } @@ -12022,8 +13754,8 @@ basic_json_parser_66: @note Currently, only `remove`, `add`, and `replace` operations are generated. - @param[in] source JSON value to copare from - @param[in] target JSON value to copare against + @param[in] source JSON value to compare from + @param[in] target JSON value to compare against @param[in] path helper value to create JSON pointers @return a JSON patch to convert the @a source to @a target @@ -12174,7 +13906,6 @@ basic_json_parser_66: /// @} }; - ///////////// // presets // ///////////// @@ -12188,7 +13919,7 @@ uses the standard template types. @since version 1.0.0 */ using json = basic_json<>; -} +} // namespace nlohmann /////////////////////// @@ -12229,7 +13960,23 @@ struct hash return h(j.dump()); } }; -} + +/// specialization for std::less +template <> +struct less<::nlohmann::detail::value_t> +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +} // namespace std /*! @brief user-defined string literal for JSON values @@ -12271,5 +14018,14 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop #endif +#if defined(__clang__) + #pragma GCC diagnostic pop +#endif + +// clean up +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_DEPRECATED #endif diff --git a/make-linux.mk b/make-linux.mk index 811d4a6e..5fb489bb 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -8,7 +8,7 @@ ifeq ($(origin CXX),default) endif INCLUDES?= -DEFS?=-D_FORTIFY_SOURCE=2 +DEFS?= LDLIBS?= DESTDIR?= @@ -55,14 +55,15 @@ endif ifeq ($(ZT_DEBUG),1) override DEFS+=-DZT_TRACE - override CFLAGS+=-Wall -g -O -pthread $(INCLUDES) $(DEFS) - override CXXFLAGS+=-Wall -g -O -std=c++11 -pthread $(INCLUDES) $(DEFS) + override CFLAGS+=-Wall -g -pthread $(INCLUDES) $(DEFS) + override CXXFLAGS+=-Wall -g -std=c++11 -pthread $(INCLUDES) $(DEFS) override LDFLAGS+= STRIP?=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! -node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CXXFLAGS=-Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else + override DEFS+=-D_FORTIFY_SOURCE=2 CFLAGS?=-O3 -fstack-protector override CFLAGS+=-Wall -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) CXXFLAGS?=-O3 -fstack-protector diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 93be64dd..f01da38e 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -72,6 +72,8 @@ public: _thingCount(0), _issuedTo(issuedTo) { + memset(_thingTypes,0,sizeof(_thingTypes)); + memset(_thingValues,0,sizeof(_thingValues)); } inline uint64_t networkId() const { return _networkId; } diff --git a/root-watcher/schema.sql b/root-watcher/schema.sql index bdb3a1cf..ade0fa3e 100644 --- a/root-watcher/schema.sql +++ b/root-watcher/schema.sql @@ -1,7 +1,6 @@ /* Schema for ZeroTier root watcher log database */ -/* If you cluster this DB using any PG clustering scheme that uses logs, you must remove UNLOGGED here! */ -CREATE UNLOGGED TABLE "Peer" +CREATE TABLE "Peer" ( "ztAddress" BIGINT NOT NULL, "timestamp" BIGINT NOT NULL, diff --git a/selftest.cpp b/selftest.cpp index 33e65f2c..e23afd6e 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -49,7 +49,6 @@ #include "osdep/OSUtils.hpp" #include "osdep/Phy.hpp" -#include "osdep/Http.hpp" #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" @@ -1019,51 +1018,6 @@ static int testPhy() return 0; } -/* -static int testHttp() -{ - std::map requestHeaders,responseHeaders; - std::string responseBody; - - InetAddress downloadZerotierDotCom; - std::vector rr(OSUtils::resolve("download.zerotier.com")); - if (rr.empty()) { - std::cout << "[http] Resolve of download.zerotier.com failed, skipping." << std::endl; - return 0; - } else { - for(std::vector::iterator r(rr.begin());r!=rr.end();++r) { - std::cout << "[http] download.zerotier.com: " << r->toString() << std::endl; - if (r->isV4()) - downloadZerotierDotCom = *r; - } - } - downloadZerotierDotCom.setPort(80); - - std::cout << "[http] GET http://download.zerotier.com/dev/1k @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - requestHeaders["Host"] = "download.zerotier.com"; - unsigned int sc = Http::GET(1024 * 1024 * 16,60000,reinterpret_cast(&downloadZerotierDotCom),"/dev/1k",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " " << responseBody.length() << " bytes "; - if (sc == 0) - std::cout << "ERROR: " << responseBody << std::endl; - else std::cout << "DONE" << std::endl; - - std::cout << "[http] GET http://download.zerotier.com/dev/4m @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - requestHeaders["Host"] = "download.zerotier.com"; - sc = Http::GET(1024 * 1024 * 16,60000,reinterpret_cast(&downloadZerotierDotCom),"/dev/4m",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " " << responseBody.length() << " bytes "; - if (sc == 0) - std::cout << "ERROR: " << responseBody << std::endl; - else std::cout << "DONE" << std::endl; - - downloadZerotierDotCom = InetAddress("1.0.0.1/1234"); - std::cout << "[http] GET @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - sc = Http::GET(1024 * 1024 * 16,2500,reinterpret_cast(&downloadZerotierDotCom),"/dev/4m",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " (should be 0, time out)" << std::endl; - - return 0; -} -*/ - #ifdef __WINDOWS__ int __cdecl _tmain(int argc, _TCHAR* argv[]) #else @@ -1127,7 +1081,6 @@ int main(int argc,char **argv) r |= testIdentity(); r |= testCertificate(); r |= testPhy(); - //r |= testHttp(); //*/ if (r) -- cgit v1.2.3 From 1b68d6dbdc5540e1b26b4ea35d019dde746af79e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 27 Apr 2017 20:47:25 -0700 Subject: License header update. --- include/ZeroTierOne.h | 10 +++++++++- node/Address.hpp | 10 +++++++++- node/Array.hpp | 10 +++++++++- node/AtomicCounter.hpp | 10 +++++++++- node/Buffer.hpp | 10 +++++++++- node/C25519.cpp | 4 +--- node/C25519.hpp | 10 +++++++++- node/Capability.cpp | 10 +++++++++- node/Capability.hpp | 10 +++++++++- node/CertificateOfMembership.cpp | 10 +++++++++- node/CertificateOfMembership.hpp | 10 +++++++++- node/CertificateOfOwnership.cpp | 10 +++++++++- node/CertificateOfOwnership.hpp | 10 +++++++++- node/CertificateOfRepresentation.hpp | 10 +++++++++- node/Cluster.cpp | 10 +++++++++- node/Cluster.hpp | 10 +++++++++- node/Constants.hpp | 10 +++++++++- node/Credential.hpp | 10 +++++++++- node/Dictionary.hpp | 10 +++++++++- node/Hashtable.hpp | 10 +++++++++- node/Identity.cpp | 10 +++++++++- node/Identity.hpp | 10 +++++++++- node/IncomingPacket.cpp | 10 +++++++++- node/IncomingPacket.hpp | 10 +++++++++- node/InetAddress.cpp | 10 +++++++++- node/InetAddress.hpp | 10 +++++++++- node/MAC.hpp | 10 +++++++++- node/Membership.cpp | 10 +++++++++- node/Membership.hpp | 10 +++++++++- node/MulticastGroup.hpp | 10 +++++++++- node/Multicaster.cpp | 10 +++++++++- node/Multicaster.hpp | 10 +++++++++- node/Mutex.hpp | 10 +++++++++- node/Network.cpp | 10 +++++++++- node/Network.hpp | 10 +++++++++- node/NetworkConfig.cpp | 10 +++++++++- node/NetworkConfig.hpp | 10 +++++++++- node/NetworkController.hpp | 10 +++++++++- node/Node.cpp | 10 +++++++++- node/Node.hpp | 10 +++++++++- node/NonCopyable.hpp | 10 +++++++++- node/OutboundMulticast.cpp | 10 +++++++++- node/OutboundMulticast.hpp | 10 +++++++++- node/Packet.cpp | 10 +++++++++- node/Packet.hpp | 10 +++++++++- node/Path.cpp | 10 +++++++++- node/Path.hpp | 10 +++++++++- node/Peer.cpp | 10 +++++++++- node/Peer.hpp | 10 +++++++++- node/Poly1305.hpp | 10 +++++++++- node/Revocation.cpp | 10 +++++++++- node/Revocation.hpp | 10 +++++++++- node/RuntimeEnvironment.hpp | 10 +++++++++- node/SHA512.cpp | 35 +++++++---------------------------- node/SHA512.hpp | 10 +++++++++- node/SelfAwareness.cpp | 10 +++++++++- node/SelfAwareness.hpp | 10 +++++++++- node/SharedPtr.hpp | 10 +++++++++- node/Switch.cpp | 10 +++++++++- node/Switch.hpp | 10 +++++++++- node/Tag.cpp | 10 +++++++++- node/Tag.hpp | 10 +++++++++- node/Topology.cpp | 10 +++++++++- node/Topology.hpp | 10 +++++++++- node/Utils.cpp | 10 +++++++++- node/Utils.hpp | 10 +++++++++- node/World.hpp | 10 +++++++++- one.cpp | 10 +++++++++- osdep/Arp.cpp | 10 +++++++++- osdep/Arp.hpp | 10 +++++++++- osdep/BSDEthernetTap.cpp | 10 +++++++++- osdep/BSDEthernetTap.hpp | 10 +++++++++- osdep/Binder.hpp | 10 +++++++++- osdep/BlockingQueue.hpp | 10 +++++++++- osdep/Http.cpp | 10 +++++++++- osdep/Http.hpp | 10 +++++++++- osdep/LinuxEthernetTap.cpp | 10 +++++++++- osdep/LinuxEthernetTap.hpp | 10 +++++++++- osdep/ManagedRoute.cpp | 10 +++++++++- osdep/ManagedRoute.hpp | 26 ++++++++++++++++++++++++++ osdep/NeighborDiscovery.cpp | 10 +++++++++- osdep/NeighborDiscovery.hpp | 10 +++++++++- osdep/OSUtils.cpp | 10 +++++++++- osdep/OSUtils.hpp | 10 +++++++++- osdep/OSXEthernetTap.cpp | 10 +++++++++- osdep/OSXEthernetTap.hpp | 10 +++++++++- osdep/Phy.hpp | 10 +++++++++- osdep/PortMapper.cpp | 10 +++++++++- osdep/PortMapper.hpp | 10 +++++++++- osdep/TestEthernetTap.hpp | 10 +++++++++- osdep/Thread.hpp | 10 +++++++++- osdep/WindowsEthernetTap.cpp | 10 +++++++++- osdep/WindowsEthernetTap.hpp | 10 +++++++++- selftest.cpp | 10 +++++++++- service/ClusterDefinition.hpp | 10 +++++++++- service/ClusterGeoIpService.cpp | 10 +++++++++- service/ClusterGeoIpService.hpp | 10 +++++++++- service/OneService.cpp | 10 +++++++++- service/OneService.hpp | 10 +++++++++- service/SoftwareUpdater.cpp | 10 +++++++++- service/SoftwareUpdater.hpp | 10 +++++++++- version.h | 10 +++++++++- 102 files changed, 925 insertions(+), 130 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 747e1855..20707a1d 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ /* diff --git a/node/Address.hpp b/node/Address.hpp index 4a5883b0..9d2d1734 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ADDRESS_HPP diff --git a/node/Array.hpp b/node/Array.hpp index 19b29eb3..5c616475 100644 --- a/node/Array.hpp +++ b/node/Array.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ARRAY_HPP diff --git a/node/AtomicCounter.hpp b/node/AtomicCounter.hpp index a0f29baa..e1864db8 100644 --- a/node/AtomicCounter.hpp +++ b/node/AtomicCounter.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ATOMICCOUNTER_HPP diff --git a/node/Buffer.hpp b/node/Buffer.hpp index 37f39e7b..ae242c73 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BUFFER_HPP diff --git a/node/C25519.cpp b/node/C25519.cpp index e9ffecc1..a78e0466 100644 --- a/node/C25519.cpp +++ b/node/C25519.cpp @@ -1,5 +1,3 @@ -// Code taken from NaCl by D. J. Bernstein and others - /* Matthew Dempsky Public domain. @@ -7,7 +5,7 @@ Derived from public domain code by D. J. Bernstein. */ // Modified very slightly for ZeroTier One by Adam Ierymenko -// (no functional changes) +// This code remains in the public domain. #include #include diff --git a/node/C25519.hpp b/node/C25519.hpp index b19d9693..da9ba665 100644 --- a/node/C25519.hpp +++ b/node/C25519.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_C25519_HPP diff --git a/node/Capability.cpp b/node/Capability.cpp index c178e566..0e02025a 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Capability.hpp" diff --git a/node/Capability.hpp b/node/Capability.hpp index 454723ac..8d4b9085 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CAPABILITY_HPP diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 9bf70216..a5445e42 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "CertificateOfMembership.hpp" diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index dfccb138..739d5390 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CERTIFICATEOFMEMBERSHIP_HPP diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp index 2bd181e0..31d0ae18 100644 --- a/node/CertificateOfOwnership.cpp +++ b/node/CertificateOfOwnership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "CertificateOfOwnership.hpp" diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index f01da38e..95039a2d 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CERTIFICATEOFOWNERSHIP_HPP diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp index 710ee577..92a71bc0 100644 --- a/node/CertificateOfRepresentation.hpp +++ b/node/CertificateOfRepresentation.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CERTIFICATEOFREPRESENTATION_HPP diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 54206f99..4d2dea76 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_ENABLE_CLUSTER diff --git a/node/Cluster.hpp b/node/Cluster.hpp index 08e32a99..74b091f5 100644 --- a/node/Cluster.hpp +++ b/node/Cluster.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CLUSTER_HPP diff --git a/node/Constants.hpp b/node/Constants.hpp index 93184efa..d3c87491 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CONSTANTS_HPP diff --git a/node/Credential.hpp b/node/Credential.hpp index 0ae2a0a8..bc81919b 100644 --- a/node/Credential.hpp +++ b/node/Credential.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CREDENTIAL_HPP diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 0db13b63..e212e453 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_DICTIONARY_HPP diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index 66f2990a..c46ed68f 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_HASHTABLE_HPP diff --git a/node/Identity.cpp b/node/Identity.cpp index d1b21e9c..ba77aa47 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Identity.hpp b/node/Identity.hpp index e4522732..b1c7d6f4 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_IDENTITY_HPP diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 303160ec..126da53c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 3d4a2e05..43a1ea10 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_INCOMINGPACKET_HPP diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 7d22eeae..62bb8145 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index c37fa621..0975a9cf 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_INETADDRESS_HPP diff --git a/node/MAC.hpp b/node/MAC.hpp index 95623f12..e7717d99 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MAC_HPP diff --git a/node/Membership.cpp b/node/Membership.cpp index 2d0471f1..466f9021 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Membership.hpp b/node/Membership.hpp index 0bc8f335..5e4475da 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MEMBERSHIP_HPP diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index be4e8084..4240db67 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MULTICASTGROUP_HPP diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 8e534b5e..52213364 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index f646a5be..2186e9c3 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MULTICASTER_HPP diff --git a/node/Mutex.hpp b/node/Mutex.hpp index d451ede0..6f1d3471 100644 --- a/node/Mutex.hpp +++ b/node/Mutex.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MUTEX_HPP diff --git a/node/Network.cpp b/node/Network.cpp index b7f25f7f..ee0f8611 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Network.hpp b/node/Network.hpp index faef0fed..cce6c41f 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NETWORK_HPP diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index fe7393e8..9effe529 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 85c24090..7bae6a91 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NETWORKCONFIG_HPP diff --git a/node/NetworkController.hpp b/node/NetworkController.hpp index 0634f435..63d44a46 100644 --- a/node/NetworkController.hpp +++ b/node/NetworkController.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NETWORKCONFIGMASTER_HPP diff --git a/node/Node.cpp b/node/Node.cpp index ccbe9411..5848d953 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Node.hpp b/node/Node.hpp index d25a619b..95587161 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NODE_HPP diff --git a/node/NonCopyable.hpp b/node/NonCopyable.hpp index 6d4daa86..25c71b1c 100644 --- a/node/NonCopyable.hpp +++ b/node/NonCopyable.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NONCOPYABLE_HPP__ diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index 285bfa5d..a2341ffd 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Constants.hpp" diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 0ecf113f..0c988804 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_OUTBOUNDMULTICAST_HPP diff --git a/node/Packet.cpp b/node/Packet.cpp index 8a57dd55..d60a3a34 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Packet.hpp b/node/Packet.hpp index 8ad2c0f9..1de679e7 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_N_PACKET_HPP diff --git a/node/Path.cpp b/node/Path.cpp index 7366b56f..a5fe1aa7 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Path.hpp" diff --git a/node/Path.hpp b/node/Path.hpp index aef628d4..32bceae0 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_PATH_HPP diff --git a/node/Peer.cpp b/node/Peer.cpp index 2e9f6a2b..01905833 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "../version.h" diff --git a/node/Peer.hpp b/node/Peer.hpp index b9d85404..9b57f23e 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_PEER_HPP diff --git a/node/Poly1305.hpp b/node/Poly1305.hpp index 62d57546..ff709983 100644 --- a/node/Poly1305.hpp +++ b/node/Poly1305.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_POLY1305_HPP diff --git a/node/Revocation.cpp b/node/Revocation.cpp index bab5653c..026058da 100644 --- a/node/Revocation.cpp +++ b/node/Revocation.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Revocation.hpp" diff --git a/node/Revocation.hpp b/node/Revocation.hpp index e5e013bd..e8f5d00d 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_REVOCATION_HPP diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 7ba1c989..d8e1d699 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_RUNTIMEENVIRONMENT_HPP diff --git a/node/SHA512.cpp b/node/SHA512.cpp index 76737d37..c8d81dd1 100644 --- a/node/SHA512.cpp +++ b/node/SHA512.cpp @@ -1,20 +1,11 @@ +// Code taken from NaCl by D. J. Bernstein and others +// Public domain + /* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ +20080913 +D. J. Bernstein +Public domain. +*/ #include #include @@ -25,18 +16,6 @@ namespace ZeroTier { -////////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -// Code taken from NaCl by D. J. Bernstein and others -// Public domain - -/* -20080913 -D. J. Bernstein -Public domain. -*/ - #define uint64 uint64_t #ifdef ZT_NO_TYPE_PUNNING diff --git a/node/SHA512.hpp b/node/SHA512.hpp index 639a7dfd..584f8e11 100644 --- a/node/SHA512.hpp +++ b/node/SHA512.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SHA512_HPP diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index cba84cdc..c5daddc3 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index c1db0c84..63c416bf 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SELFAWARENESS_HPP diff --git a/node/SharedPtr.hpp b/node/SharedPtr.hpp index 1dd3b43d..09010f67 100644 --- a/node/SharedPtr.hpp +++ b/node/SharedPtr.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SHAREDPTR_HPP diff --git a/node/Switch.cpp b/node/Switch.cpp index 56299a9a..211b706a 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Switch.hpp b/node/Switch.hpp index ff350934..9793dd45 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_N_SWITCH_HPP diff --git a/node/Tag.cpp b/node/Tag.cpp index 3f924da1..39b17f2a 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Tag.hpp" diff --git a/node/Tag.hpp b/node/Tag.hpp index 1f7f6835..746ade26 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_TAG_HPP diff --git a/node/Topology.cpp b/node/Topology.cpp index a1d37332..80f4ed4e 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Constants.hpp" diff --git a/node/Topology.hpp b/node/Topology.hpp index d29c424e..d06ba94b 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_TOPOLOGY_HPP diff --git a/node/Utils.cpp b/node/Utils.cpp index 9ce1bf05..d69e5335 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/node/Utils.hpp b/node/Utils.hpp index ceb29d7e..25a90055 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_UTILS_HPP diff --git a/node/World.hpp b/node/World.hpp index 6e835bec..003d70e3 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_WORLD_HPP diff --git a/one.cpp b/one.cpp index b40e28fc..1f38361f 100644 --- a/one.cpp +++ b/one.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/Arp.cpp b/osdep/Arp.cpp index fcc122f0..c06f459b 100644 --- a/osdep/Arp.cpp +++ b/osdep/Arp.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/Arp.hpp b/osdep/Arp.hpp index 5f0d199a..e26fcdb3 100644 --- a/osdep/Arp.hpp +++ b/osdep/Arp.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ARP_HPP diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index 62fabc48..87a9aece 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/BSDEthernetTap.hpp b/osdep/BSDEthernetTap.hpp index 8c6314db..3cb9c10e 100644 --- a/osdep/BSDEthernetTap.hpp +++ b/osdep/BSDEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BSDETHERNETTAP_HPP diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index 9829f170..ee832825 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BINDER_HPP diff --git a/osdep/BlockingQueue.hpp b/osdep/BlockingQueue.hpp index 6172f4da..34abcb67 100644 --- a/osdep/BlockingQueue.hpp +++ b/osdep/BlockingQueue.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BLOCKINGQUEUE_HPP diff --git a/osdep/Http.cpp b/osdep/Http.cpp index 064ccd0c..d2540071 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/Http.hpp b/osdep/Http.hpp index e7d4d03e..3f98d760 100644 --- a/osdep/Http.hpp +++ b/osdep/Http.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_HTTP_HPP diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index f74efc0a..2d3891e3 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp index a2a00a79..ab9d2370 100644 --- a/osdep/LinuxEthernetTap.hpp +++ b/osdep/LinuxEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_LINUXETHERNETTAP_HPP diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 3a020d61..fca1c290 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "../node/Constants.hpp" diff --git a/osdep/ManagedRoute.hpp b/osdep/ManagedRoute.hpp index fd77a79a..849bddf5 100644 --- a/osdep/ManagedRoute.hpp +++ b/osdep/ManagedRoute.hpp @@ -1,3 +1,29 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + #ifndef ZT_MANAGEDROUTE_HPP #define ZT_MANAGEDROUTE_HPP diff --git a/osdep/NeighborDiscovery.cpp b/osdep/NeighborDiscovery.cpp index 4f636310..cd8b9b91 100644 --- a/osdep/NeighborDiscovery.cpp +++ b/osdep/NeighborDiscovery.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "NeighborDiscovery.hpp" diff --git a/osdep/NeighborDiscovery.hpp b/osdep/NeighborDiscovery.hpp index 47831bda..2e7a68ba 100644 --- a/osdep/NeighborDiscovery.hpp +++ b/osdep/NeighborDiscovery.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NEIGHBORDISCOVERY_HPP diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index fd5efed0..b7fce982 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index b84d5d2d..4b9ee893 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_OSUTILS_HPP diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index f70908b8..53c9ba98 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/OSXEthernetTap.hpp b/osdep/OSXEthernetTap.hpp index 5a96c210..ed7f39c3 100644 --- a/osdep/OSXEthernetTap.hpp +++ b/osdep/OSXEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_OSXETHERNETTAP_HPP diff --git a/osdep/Phy.hpp b/osdep/Phy.hpp index eab8a317..01a339e9 100644 --- a/osdep/Phy.hpp +++ b/osdep/Phy.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_PHY_HPP diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index d3a19384..99286172 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_USE_MINIUPNPC diff --git a/osdep/PortMapper.hpp b/osdep/PortMapper.hpp index 0b8d15fc..61015a09 100644 --- a/osdep/PortMapper.hpp +++ b/osdep/PortMapper.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_USE_MINIUPNPC diff --git a/osdep/TestEthernetTap.hpp b/osdep/TestEthernetTap.hpp index 6c044a94..afd89541 100644 --- a/osdep/TestEthernetTap.hpp +++ b/osdep/TestEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_TESTETHERNETTAP_HPP diff --git a/osdep/Thread.hpp b/osdep/Thread.hpp index 5423a8ab..a2f0919f 100644 --- a/osdep/Thread.hpp +++ b/osdep/Thread.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_THREAD_HPP diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index 79b9d35e..c37c7410 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/osdep/WindowsEthernetTap.hpp b/osdep/WindowsEthernetTap.hpp index f2cf73f3..a3c1c0c3 100644 --- a/osdep/WindowsEthernetTap.hpp +++ b/osdep/WindowsEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_WINDOWSETHERNETTAP_HPP diff --git a/selftest.cpp b/selftest.cpp index e23afd6e..209fe203 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/service/ClusterDefinition.hpp b/service/ClusterDefinition.hpp index dda1a8c8..9947e46b 100644 --- a/service/ClusterDefinition.hpp +++ b/service/ClusterDefinition.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CLUSTERDEFINITION_HPP diff --git a/service/ClusterGeoIpService.cpp b/service/ClusterGeoIpService.cpp index 89015c51..2dcc9179 100644 --- a/service/ClusterGeoIpService.cpp +++ b/service/ClusterGeoIpService.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_ENABLE_CLUSTER diff --git a/service/ClusterGeoIpService.hpp b/service/ClusterGeoIpService.hpp index ff2fcdb8..380f944f 100644 --- a/service/ClusterGeoIpService.hpp +++ b/service/ClusterGeoIpService.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CLUSTERGEOIPSERVICE_HPP diff --git a/service/OneService.cpp b/service/OneService.cpp index 988e723d..9f9cec0a 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/service/OneService.hpp b/service/OneService.hpp index 3390f2ac..f52cd40e 100644 --- a/service/OneService.hpp +++ b/service/OneService.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ONESERVICE_HPP diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 7ec377cc..d94beab5 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index 4bb0ef51..ff3e36df 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SOFTWAREUPDATER_HPP diff --git a/version.h b/version.h index c51bfee2..b3b2fc81 100644 --- a/version.h +++ b/version.h @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef _ZT_VERSION_H -- cgit v1.2.3 From 718e1d6c082453bfbab8b900f5ffde42047fc814 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 1 May 2017 13:21:26 -0700 Subject: Finish removing constantly changing stuff from controller. --- controller/EmbeddedNetworkController.cpp | 160 ++++++++++++++++++------------- controller/EmbeddedNetworkController.hpp | 34 ++++++- controller/JSONDB.hpp | 13 +++ controller/README.md | 22 ++--- node/Dictionary.hpp | 2 + 5 files changed, 143 insertions(+), 88 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 70640ff5..3ec02454 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -518,7 +518,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( json member; if (!_db.getNetworkMember(nwid,address,member)) return 404; - _addMemberNonPersistedFields(member,OSUtils::now()); + _addMemberNonPersistedFields(nwid,address,member,OSUtils::now()); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; } else { @@ -569,6 +569,26 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } // else 404 + } else if ((path.size() == 1)&&(path[0] == "memberStatus")) { + + const uint64_t now = OSUtils::now(); + Mutex::Lock _l(_memberStatus_m); + responseBody.push_back('{'); + _db.eachId([this,&responseBody,&now](uint64_t networkId,uint64_t nodeId) { + char tmp[64]; + auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); + Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%s", + (responseBody.length() > 1) ? "," : "", + (unsigned long long)networkId, + (unsigned long long)nodeId, + ((ms != _memberStatus.end())&&(ms->second.online(now))) ? "true" : "false"); + responseBody.append(tmp); + }); + responseBody.push_back('}'); + responseContentType = "application/json"; + + return 200; + } else { char tmp[4096]; @@ -649,10 +669,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!newAuth) { Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); - Mutex::Lock _l(_lastRequestTime_m); - for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { - if ((now - i->second) < ZT_NETWORK_AUTOCONF_DELAY) - _node->ncSendRevocation(Address(i->first.first),rev); + + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == nwid)&&(i->second.online(now))) + _node->ncSendRevocation(Address(i->first.nodeId),rev); } } } @@ -720,10 +741,17 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetworkMember(nwid,address,member); - _pushMemberUpdate(now,nwid,member); + + // Push update to member if online + try { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,address)]; + if ((ms.online(now))&&(ms.lastRequestMetaData)) + request(nwid,InetAddress(),0,ms.identity,ms.lastRequestMetaData); + } catch ( ... ) {} } - _addMemberNonPersistedFields(member,now); + _addMemberNonPersistedFields(nwid,address,member,now); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; @@ -1022,10 +1050,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); _db.saveNetwork(nwid,network); - // Send an update to all members of the network - _db.eachMember(nwid,[this,&now,&nwid](uint64_t networkId,uint64_t nodeId,const json &obj) { - this->_pushMemberUpdate(now,nwid,obj); - }); + // Send an update to all members of the network that are online + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == nwid)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) + request(nwid,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData); + } } JSONDB::NetworkSummaryInfo ns; @@ -1044,7 +1074,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json testRec; const uint64_t now = OSUtils::now(); testRec["clock"] = now; - testRec["uptime"] = (now - _startTime); + testRec["startTime"] = _startTime; testRec["content"] = b; responseBody = OSUtils::jsonDump(testRec); _db.writeRaw("pong",responseBody); @@ -1075,6 +1105,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( const uint64_t address = Utils::hexStrToU64(path[3].c_str()); json member = _db.eraseNetworkMember(nwid,address); + + { + Mutex::Lock _l(_memberStatus_m); + _memberStatus.erase(_MemberStatusKey(nwid,address)); + } + if (!member.size()) return 404; responseBody = OSUtils::jsonDump(member); @@ -1083,6 +1119,16 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( } } else { json network = _db.eraseNetwork(nwid); + + { + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();) { + if (i->first.networkId == nwid) + _memberStatus.erase(i++); + else ++i; + } + } + if (!network.size()) return 404; responseBody = OSUtils::jsonDump(network); @@ -1106,6 +1152,7 @@ void EmbeddedNetworkController::threadMain() _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); } catch ( ... ) {} delete qe; + if (_running) { uint64_t now = OSUtils::now(); if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { @@ -1196,11 +1243,11 @@ void EmbeddedNetworkController::_request( const uint64_t now = OSUtils::now(); if (requestPacketId) { - Mutex::Lock _l(_lastRequestTime_m); - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; + if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) return; - lrt = now; + ms.lastRequestTime = now; } Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); @@ -1315,37 +1362,37 @@ void EmbeddedNetworkController::_request( member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } - // Log this request - if (requestPacketId) { // only log if this is a request, not for generated pushes - json rlEntry = json::object(); - rlEntry["ts"] = now; - rlEntry["auth"] = (authorizedBy) ? true : false; - rlEntry["authBy"] = (authorizedBy) ? authorizedBy : ""; - rlEntry["vMajor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["vMinor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["vRev"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["vProto"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - if (fromAddr) - rlEntry["fromAddr"] = fromAddr.toString(); - - json recentLog = json::array(); - recentLog.push_back(rlEntry); - json &oldLog = member["recentLog"]; - if (oldLog.is_array()) { - for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - break; - } - } - member["recentLog"] = recentLog; + if (authorizedBy) { + // Update version info and meta-data if authorized and if this is a genuine request + if (requestPacketId) { + const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + const uint64_t vMinor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + const uint64_t vRev = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + const uint64_t vProto = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - // Also only do this on real requests - member["lastRequestMetaData"] = metaData.data(); - } + member["vMajor"] = vMajor; + member["vMinor"] = vMinor; + member["vRev"] = vRev; + member["vProto"] = vProto; - // If they are not authorized, STOP! - if (!authorizedBy) { + { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; + + ms.vMajor = (int)vMajor; + ms.vMinor = (int)vMinor; + ms.vRev = (int)vRev; + ms.vProto = (int)vProto; + ms.lastRequestMetaData = metaData; + ms.identity = identity; + + if (fromAddr) + ms.physicalAddr = fromAddr; + member["physicalAddr"] = ms.physicalAddr.toString(); + } + } + } else { + // If they are not authorized, STOP! _removeMemberNonPersistedFields(member); if (origMember != member) _db.saveNetworkMember(nwid,identity.address().toInt(),member); @@ -1702,27 +1749,4 @@ void EmbeddedNetworkController::_request( _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } -void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member) -{ - try { - const std::string idstr = member["identity"]; - const std::string mdstr = member["lastRequestMetaData"]; - if ((idstr.length() > 0)&&(mdstr.length() > 0)) { - const Identity id(idstr); - bool online; - { - Mutex::Lock _l(_lastRequestTime_m); - std::map< std::pair,uint64_t >::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); - online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); - } - if (online) { - Dictionary metaData(mdstr.c_str()); - try { - this->request(nwid,InetAddress(),0,id,metaData); - } catch ( ... ) {} - } - } - } catch ( ... ) {} -} - } // namespace ZeroTier diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 64ea6884..faf7f029 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -107,7 +107,6 @@ private: static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); - void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); // These init objects with default and static/informational fields inline void _initMember(nlohmann::json &member) @@ -164,9 +163,11 @@ private: network.erase("activeMemberCount"); network.erase("totalMemberCount"); } - inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) + inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,uint64_t now) { member["clock"] = now; + Mutex::Lock _l(_memberStatus_m); + member["online"] = _memberStatus[_MemberStatusKey(nwid,nodeId)].online(now); } inline void _removeMemberNonPersistedFields(nlohmann::json &member) { @@ -191,8 +192,33 @@ private: std::list< ZT_CircuitTest > _tests; Mutex _tests_m; - std::map< std::pair,uint64_t > _lastRequestTime; // last request time by - Mutex _lastRequestTime_m; + struct _MemberStatusKey + { + _MemberStatusKey() : networkId(0),nodeId(0) {} + _MemberStatusKey(const uint64_t nwid,const uint64_t nid) : networkId(nwid),nodeId(nid) {} + uint64_t networkId; + uint64_t nodeId; + inline bool operator==(const _MemberStatusKey &k) const { return ((k.networkId == networkId)&&(k.nodeId == nodeId)); } + }; + struct _MemberStatus + { + _MemberStatus() : lastRequestTime(0),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {} + uint64_t lastRequestTime; + int vMajor,vMinor,vRev,vProto; + Dictionary lastRequestMetaData; + Identity identity; + InetAddress physicalAddr; // last known physical address + inline bool online(const uint64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } + }; + struct _MemberStatusHash + { + inline std::size_t operator()(const _MemberStatusKey &networkIdNodeId) const + { + return (std::size_t)(networkIdNodeId.networkId + networkIdNodeId.nodeId); + } + }; + std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus; + Mutex _memberStatus_m; }; } // namespace ZeroTier diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index ba16d97b..530f9632 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -107,6 +107,19 @@ public: } } + template + inline void eachId(F func) + { + Mutex::Lock _l(_networks_m); + for(std::unordered_map::const_iterator i(_networks.begin());i!=_networks.end();++i) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + try { + func(i->first,m->first); + } catch ( ... ) {} + } + } + } + void threadMain() throw(); diff --git a/controller/README.md b/controller/README.md index db8d0153..3519eb11 100644 --- a/controller/README.md +++ b/controller/README.md @@ -227,22 +227,12 @@ This returns an object containing all currently online members and the most rece | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | -| memberRevision | integer | Member revision counter | no | -| recentLog | array[object] | Recent member activity log; see below | no | +| revision | integer | Member revision counter | no | +| vMajor | integer | Most recently known major version | no | +| vMinor | integer | Most recently known minor version | no | +| vRev | integer | Most recently known revision | no | +| vProto | integer | Most recently known protocl version | no | +| physicalAddr | string | Last known physical IP/port or null if none | no | Note that managed IP assignments are only used if they fall within a managed route. Otherwise they are ignored. -**Recent log object format:** - -| Field | Type | Description | -| --------------------- | ------------- | ------------------------------------------------- | -| ts | integer | Time of request, ms since epoch | -| auth | boolean | Was member authorized? | -| authBy | string | How was member authorized? | -| vMajor | integer | Client major version or -1 if unknown | -| vMinor | integer | Client minor version or -1 if unknown | -| vRev | integer | Client revision or -1 if unknown | -| vProto | integer | ZeroTier protocol version reported by client | -| fromAddr | string | Physical address if known | - -The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index e212e453..4413d628 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -99,6 +99,8 @@ public: return *this; } + inline operator bool() const { return (_d[0] != 0); } + /** * Load a dictionary from a C-string * -- cgit v1.2.3 From 41c187ba12fc05f6e9ccd5f8acbc248c2a3d16e1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 3 May 2017 07:43:23 -0700 Subject: Another very small crypto optimization. --- node/Salsa20.hpp | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 52592602..bfb6d9d9 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -48,6 +48,43 @@ public: static inline void memxor(uint8_t *d,const uint8_t *s,unsigned int len) { #ifdef ZT_SALSA20_SSE + while (len >= 128) { + __m128i s0 = _mm_loadu_si128(reinterpret_cast(s)); + __m128i s1 = _mm_loadu_si128(reinterpret_cast(s + 16)); + __m128i s2 = _mm_loadu_si128(reinterpret_cast(s + 32)); + __m128i s3 = _mm_loadu_si128(reinterpret_cast(s + 48)); + __m128i s4 = _mm_loadu_si128(reinterpret_cast(s + 64)); + __m128i s5 = _mm_loadu_si128(reinterpret_cast(s + 80)); + __m128i s6 = _mm_loadu_si128(reinterpret_cast(s + 96)); + __m128i s7 = _mm_loadu_si128(reinterpret_cast(s + 112)); + __m128i d0 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d)); + __m128i d1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 16)); + __m128i d2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 32)); + __m128i d3 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 48)); + __m128i d4 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 64)); + __m128i d5 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 80)); + __m128i d6 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 96)); + __m128i d7 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 112)); + d0 = _mm_xor_si128(d0,s0); + d1 = _mm_xor_si128(d1,s1); + d2 = _mm_xor_si128(d2,s2); + d3 = _mm_xor_si128(d3,s3); + d4 = _mm_xor_si128(d4,s4); + d5 = _mm_xor_si128(d5,s5); + d6 = _mm_xor_si128(d6,s6); + d7 = _mm_xor_si128(d7,s7); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d),d0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 16),d1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 32),d2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 48),d3); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 64),d4); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 80),d5); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 96),d6); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 112),d7); + s += 128; + d += 128; + len -= 128; + } while (len >= 16) { _mm_storeu_si128(reinterpret_cast<__m128i *>(d),_mm_xor_si128(_mm_loadu_si128(reinterpret_cast<__m128i *>(d)),_mm_loadu_si128(reinterpret_cast(s)))); s += 16; @@ -67,8 +104,10 @@ public: } #endif #endif - while (len--) + while (len) { + --len; *(d++) ^= *(s++); + } } /** -- cgit v1.2.3 From ceeb8ee0bcd591d79dc61a684ede2c9cae91323c Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 4 May 2017 15:25:48 -0700 Subject: added isEqualPrefix to InetAddress --- node/InetAddress.cpp | 24 ++++++++++++++++++++++++ node/InetAddress.hpp | 10 ++++++++++ 2 files changed, 34 insertions(+) (limited to 'node') diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 62bb8145..0fbb2d68 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -287,6 +287,30 @@ InetAddress InetAddress::network() const return r; } +#ifdef ZT_SDK + bool InetAddress::isEqualPrefix(const InetAddress &addr) const + { + if (addr.ss_family == ss_family) { + switch(ss_family) { + case AF_INET6: { + const InetAddress mask(netmask()); + InetAddress addr_mask(addr.netmask()); + const uint8_t *n = reinterpret_cast(reinterpret_cast(&addr_mask)->sin6_addr.s6_addr); + const uint8_t *m = reinterpret_cast(reinterpret_cast(&mask)->sin6_addr.s6_addr); + const uint8_t *a = reinterpret_cast(reinterpret_cast(&addr)->sin6_addr.s6_addr); + const uint8_t *b = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); + for(unsigned int i=0;i<16;++i) { + if ((a[i] & m[i]) != (b[i] & n[i])) + return false; + } + return true; + } + } + } + return false; + } +#endif + bool InetAddress::containsAddress(const InetAddress &addr) const { if (addr.ss_family == ss_family) { diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 0975a9cf..4cb9a4dc 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -355,6 +355,16 @@ struct InetAddress : public sockaddr_storage */ InetAddress network() const; +#ifdef ZT_SDK + /** + * Test whether this IPv6 prefix matches the prefix of a given IPv6 address + * + * @param addr Address to check + * @return True if this IPv6 prefix matches the prefix of a given IPv6 address + */ + bool isEqualPrefix(const InetAddress &addr) const; +#endif + /** * Test whether this IP/netmask contains this address * -- cgit v1.2.3 From 107e3e41065a816354c3f383736c5abbb156b0d3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 May 2017 17:12:02 -0700 Subject: First pass of configurable MTU and max MTU increase. --- include/ZeroTierOne.h | 22 +--------------------- node/Constants.hpp | 9 +++------ node/IncomingPacket.cpp | 2 +- node/Network.cpp | 4 +++- node/NetworkConfig.cpp | 7 +++++++ node/NetworkConfig.hpp | 7 +++++++ 6 files changed, 22 insertions(+), 29 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 20707a1d..21adbe02 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -61,28 +61,8 @@ extern "C" { /** * Maximum MTU for ZeroTier virtual networks - * - * This is pretty much an unchangeable global constant. To make it change - * across nodes would require logic to send ICMP packet too big messages, - * which would complicate things. 1500 has been good enough on most LANs - * for ages, so a larger MTU should be fine for the forseeable future. This - * typically results in two UDP packets per single large frame. Experimental - * results seem to show that this is good. Larger MTUs resulting in more - * fragments seemed too brittle on slow/crummy links for no benefit. - * - * If this does change, also change it in tap.h in the tuntaposx code under - * mac-tap. - * - * Overhead for a normal frame split into two packets: - * - * 1414 = 1444 (typical UDP MTU) - 28 (packet header) - 2 (ethertype) - * 1428 = 1444 (typical UDP MTU) - 16 (fragment header) - * SUM: 2842 - * - * We use 2800, which leaves some room for other payload in other types of - * messages such as multicast propagation or future support for bridging. */ -#define ZT_MAX_MTU 2800 +#define ZT_MAX_MTU 10000 /** * Maximum length of network short name diff --git a/node/Constants.hpp b/node/Constants.hpp index d3c87491..8aeaef02 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -159,15 +159,12 @@ /** * Default MTU used for Ethernet tap device */ -#define ZT_IF_MTU ZT_MAX_MTU +#define ZT_DEFAULT_MTU 2800 /** - * Maximum number of packet fragments we'll support - * - * The actual spec allows 16, but this is the most we'll support right - * now. Packets with more than this many fragments are dropped. + * Maximum number of packet fragments we'll support (protocol max: 16) */ -#define ZT_MAX_PACKET_FRAGMENTS 4 +#define ZT_MAX_PACKET_FRAGMENTS 7 /** * Size of RX queue diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 126da53c..7ef2054b 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1125,7 +1125,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, //TRACE("<address().toString().c_str(),flags,frameLen); - if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) { + if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) { if (!to.mac().isMulticast()) { TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay diff --git a/node/Network.cpp b/node/Network.cpp index ee0f8611..de2ea7d7 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1225,6 +1225,8 @@ void Network::requestConfiguration(void *tPtr) nconf->revision = 1; nconf->issuedTo = RR->identity.address(); nconf->flags = ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + nconf->mtu = ZT_DEFAULT_MTU; + nconf->multicastLimit = 0; nconf->staticIpCount = 1; nconf->ruleCount = 14; nconf->staticIps[0] = InetAddress::makeIpv66plane(_id,RR->identity.address().toInt()); @@ -1495,7 +1497,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const else ec->name[0] = (char)0; ec->status = _status(); ec->type = (_config) ? (_config.isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE; - ec->mtu = ZT_IF_MTU; + ec->mtu = (_config) ? _config.mtu : ZT_DEFAULT_MTU; ec->physicalMtu = ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 16); ec->dhcp = 0; std::vector
ab(_config.activeBridges()); diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 9effe529..c39f6cab 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -51,6 +51,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MTU,(uint64_t)this->mtu)) return false; #ifdef ZT_SUPPORT_OLD_STYLE_NETCONF if (includeLegacy) { @@ -217,6 +218,12 @@ bool NetworkConfig::fromDictionary(const DictionarymulticastLimit = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,0); d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name,sizeof(this->name)); + this->mtu = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MTU,ZT_DEFAULT_MTU); + if (this->mtu < 1280) + this->mtu = 1280; // minimum MTU allowed by IPv6 standard and others + else if (this->mtu > ZT_MAX_MTU) + this->mtu = ZT_MAX_MTU; + if (d.getUI(ZT_NETWORKCONFIG_DICT_KEY_VERSION,0) < 6) { #ifdef ZT_SUPPORT_OLD_STYLE_NETCONF char tmp2[1024]; diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 7bae6a91..fdd078d5 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -167,6 +167,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_TYPE "t" // text #define ZT_NETWORKCONFIG_DICT_KEY_NAME "n" +// network MTU +#define ZT_NETWORKCONFIG_DICT_KEY_MTU "mtu" // credential time max delta in ms #define ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA "ctmd" // binary serialized certificate of membership @@ -465,6 +467,11 @@ public: */ uint64_t flags; + /** + * Network MTU + */ + unsigned int mtu; + /** * Maximum number of recipients per multicast (not including active bridges) */ -- cgit v1.2.3 From b9c1407013eba0f26f311ab97937048eaa0ce9df Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 May 2017 09:36:37 -0700 Subject: Adjust PUSH_DIRECT_PATH circuit breaker, and comment out traces to reduce noise. --- node/Constants.hpp | 6 +++--- node/IncomingPacket.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 8aeaef02..494ebace 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -369,7 +369,7 @@ /** * Time horizon for push direct paths cutoff */ -#define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000 +#define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 30000 /** * Maximum number of direct path pushes within cutoff time @@ -378,12 +378,12 @@ * per CUTOFF_TIME milliseconds per peer to prevent this from being * useful for DOS amplification attacks. */ -#define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5 +#define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 8 /** * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6) */ -#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 4 +#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 8 /** * Time horizon for VERB_NETWORK_CREDENTIALS cutoff diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 7ef2054b..131659f9 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1185,7 +1185,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { - TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + //TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1220,7 +1220,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { - TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); + //TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } } } break; @@ -1237,7 +1237,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { - TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); + //TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } } } break; -- cgit v1.2.3 From f479b76772ecefde3e01fd5c2933ed0536922fd6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 May 2017 20:22:08 -0700 Subject: define likely/unlikely --- node/Constants.hpp | 18 ++++++++++++++++++ node/Packet.cpp | 2 ++ 2 files changed, 20 insertions(+) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 494ebace..3974f0ec 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -60,6 +60,8 @@ #endif #ifdef __APPLE__ +#define likely(x) __builtin_expect((x),1) +#define unlikely(x) __builtin_expect((x),0) #include #ifndef __UNIX_LIKE__ #define __UNIX_LIKE__ @@ -132,6 +134,22 @@ #include #endif +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +#ifndef likely +#define likely(x) __builtin_expect((x),1) +#endif +#ifndef unlikely +#define unlikely(x) __builtin_expect((x),0) +#endif +#else +#ifndef likely +#define likely(x) (x) +#endif +#ifndef unlikely +#define unlikely(x) (x) +#endif +#endif + /** * Length of a ZeroTier address in bytes */ diff --git a/node/Packet.cpp b/node/Packet.cpp index d60a3a34..e778e3bb 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -320,6 +320,7 @@ union LZ4_streamDecode_u { #define FORCE_INLINE static inline #endif +#if 0 #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else @@ -328,6 +329,7 @@ union LZ4_streamDecode_u { #define likely(expr) expect((expr) != 0, 1) #define unlikely(expr) expect((expr) != 0, 0) +#endif /*-************************************ * Memory routines -- cgit v1.2.3 From 2d74c60d47e03abfcaee81ab55dcfff6241e68a0 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 9 May 2017 21:54:23 -0700 Subject: Add branch hints to bounds checking in Buffer. --- node/Buffer.hpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'node') diff --git a/node/Buffer.hpp b/node/Buffer.hpp index ae242c73..8e6b78fd 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -123,7 +123,7 @@ public: inline Buffer &operator=(const Buffer &b) throw(std::out_of_range) { - if (b._l > C) + if (unlikely(b._l > C)) throw std::out_of_range("Buffer: assignment from buffer larger than capacity"); memcpy(_b,b._b,_l = b._l); return *this; @@ -139,7 +139,7 @@ public: inline void copyFrom(const void *b,unsigned int l) throw(std::out_of_range) { - if (l > C) + if (unlikely(l > C)) throw std::out_of_range("Buffer: set from C array larger than capacity"); _l = l; memcpy(_b,b,l); @@ -148,7 +148,7 @@ public: unsigned char operator[](const unsigned int i) const throw(std::out_of_range) { - if (i >= _l) + if (unlikely(i >= _l)) throw std::out_of_range("Buffer: [] beyond end of data"); return (unsigned char)_b[i]; } @@ -156,7 +156,7 @@ public: unsigned char &operator[](const unsigned int i) throw(std::out_of_range) { - if (i >= _l) + if (unlikely(i >= _l)) throw std::out_of_range("Buffer: [] beyond end of data"); return ((unsigned char *)_b)[i]; } @@ -177,14 +177,14 @@ public: unsigned char *field(unsigned int i,unsigned int l) throw(std::out_of_range) { - if ((i + l) > _l) + if (unlikely((i + l) > _l)) throw std::out_of_range("Buffer: field() beyond end of data"); return (unsigned char *)(_b + i); } const unsigned char *field(unsigned int i,unsigned int l) const throw(std::out_of_range) { - if ((i + l) > _l) + if (unlikely((i + l) > _l)) throw std::out_of_range("Buffer: field() beyond end of data"); return (const unsigned char *)(_b + i); } @@ -200,7 +200,7 @@ public: inline void setAt(unsigned int i,const T v) throw(std::out_of_range) { - if ((i + sizeof(T)) > _l) + if (unlikely((i + sizeof(T)) > _l)) throw std::out_of_range("Buffer: setAt() beyond end of data"); #ifdef ZT_NO_TYPE_PUNNING uint8_t *p = reinterpret_cast(_b + i); @@ -223,7 +223,7 @@ public: inline T at(unsigned int i) const throw(std::out_of_range) { - if ((i + sizeof(T)) > _l) + if (unlikely((i + sizeof(T)) > _l)) throw std::out_of_range("Buffer: at() beyond end of data"); #ifdef ZT_NO_TYPE_PUNNING T v = 0; @@ -250,7 +250,7 @@ public: inline void append(const T v) throw(std::out_of_range) { - if ((_l + sizeof(T)) > C) + if (unlikely((_l + sizeof(T)) > C)) throw std::out_of_range("Buffer: append beyond capacity"); #ifdef ZT_NO_TYPE_PUNNING uint8_t *p = reinterpret_cast(_b + _l); @@ -273,7 +273,7 @@ public: inline void append(unsigned char c,unsigned int n) throw(std::out_of_range) { - if ((_l + n) > C) + if (unlikely((_l + n) > C)) throw std::out_of_range("Buffer: append beyond capacity"); for(unsigned int i=0;i C) + if (unlikely((_l + l) > C)) throw std::out_of_range("Buffer: append beyond capacity"); memcpy(_b + _l,b,l); _l += l; @@ -317,7 +317,7 @@ public: throw(std::out_of_range) { for(;;) { - if (_l >= C) + if (unlikely(_l >= C)) throw std::out_of_range("Buffer: append beyond capacity"); if (!(_b[_l++] = *(s++))) break; @@ -351,7 +351,7 @@ public: inline char *appendField(unsigned int l) throw(std::out_of_range) { - if ((_l + l) > C) + if (unlikely((_l + l) > C)) throw std::out_of_range("Buffer: append beyond capacity"); char *r = _b + _l; _l += l; @@ -369,7 +369,7 @@ public: inline void addSize(unsigned int i) throw(std::out_of_range) { - if ((i + _l) > C) + if (unlikely((i + _l) > C)) throw std::out_of_range("Buffer: setSize to larger than capacity"); _l += i; } @@ -385,7 +385,7 @@ public: inline void setSize(const unsigned int i) throw(std::out_of_range) { - if (i > C) + if (unlikely(i > C)) throw std::out_of_range("Buffer: setSize to larger than capacity"); _l = i; } @@ -401,7 +401,7 @@ public: { if (!at) return; - if (at > _l) + if (unlikely(at > _l)) throw std::out_of_range("Buffer: behead() beyond capacity"); ::memmove(_b,_b + at,_l -= at); } @@ -417,7 +417,7 @@ public: throw(std::out_of_range) { const unsigned int endr = at + length; - if (endr > _l) + if (unlikely(endr > _l)) throw std::out_of_range("Buffer: erase() range beyond end of buffer"); ::memmove(_b + at,_b + endr,_l - endr); _l -= length; -- cgit v1.2.3 From 5e6a2a17b02904b5349f379ff6f6d91a4c849067 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 19 May 2017 15:32:52 -0700 Subject: Cluster build fix. --- node/Cluster.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Cluster.cpp b/node/Cluster.cpp index 4d2dea76..119aec29 100644 --- a/node/Cluster.cpp +++ b/node/Cluster.cpp @@ -257,7 +257,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) memcpy(keytmp,_key,32); for(int i=0;i<8;++i) keytmp[i] ^= reinterpret_cast(msg)[i]; - Salsa20 s20(keytmp,256,reinterpret_cast(msg) + 8); + Salsa20 s20(keytmp,reinterpret_cast(msg) + 8); Utils::burn(keytmp,sizeof(keytmp)); // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") @@ -956,7 +956,7 @@ void Cluster::_flush(uint16_t memberId) memcpy(keytmp,m.key,32); for(int i=0;i<8;++i) keytmp[i] ^= m.q[i]; - Salsa20 s20(keytmp,256,m.q.field(8,8)); + Salsa20 s20(keytmp,m.q.field(8,8)); Utils::burn(keytmp,sizeof(keytmp)); // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") -- cgit v1.2.3 From f9a9c2d009c719c5dfcb1897b9a9a48cf6b17487 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 23 May 2017 14:45:16 -0700 Subject: Config object was never even being initialized on leave. Never noticed since desktop and server clients did not use. --- node/Node.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 5848d953..6d7eea43 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -327,6 +327,7 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) } else { if (uptr) *uptr = *n->second->userPtr(); + n->second->externalConfig(&ctmp); n->second->destroy(); nUserPtr = n->second->userPtr(); } -- cgit v1.2.3 From 2ec88e800877cfbc7f007d21f10429bc1b493006 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 30 May 2017 10:19:45 -0700 Subject: Remove old circuit test code. Rules engine will let us do this much better and more simply. --- controller/EmbeddedNetworkController.cpp | 125 ----------------- controller/EmbeddedNetworkController.hpp | 7 - include/ZeroTierOne.h | 226 ------------------------------- node/IncomingPacket.cpp | 196 +-------------------------- node/IncomingPacket.hpp | 2 - node/Node.cpp | 88 ------------ node/Node.hpp | 6 - node/Packet.cpp | 2 - node/Packet.hpp | 116 +--------------- 9 files changed, 4 insertions(+), 764 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 3e9a28b8..e2eaa788 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -726,59 +726,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( responseContentType = "application/json"; return 200; - } else if ((path.size() == 3)&&(path[2] == "test")) { - - Mutex::Lock _l(_tests_m); - - _tests.push_back(ZT_CircuitTest()); - ZT_CircuitTest *const test = &(_tests.back()); - memset(test,0,sizeof(ZT_CircuitTest)); - - Utils::getSecureRandom(&(test->testId),sizeof(test->testId)); - test->credentialNetworkId = nwid; - test->ptr = (void *)this; - json hops = b["hops"]; - if (hops.is_array()) { - for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - } - ++test->hopCount; - } else if (hops2.is_string()) { - std::string s = hops2; - test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - ++test->hopCount; - } - } - } - test->reportAtEveryHop = (OSUtils::jsonBool(b["reportAtEveryHop"],true) ? 1 : 0); - - if (!test->hopCount) { - _tests.pop_back(); - responseBody = "{ \"message\": \"a test must contain at least one hop\" }"; - responseContentType = "application/json"; - return 400; - } - - test->timestamp = OSUtils::now(); - - if (_node) { - _node->circuitTestBegin((void *)0,test,&(EmbeddedNetworkController::_circuitTestCallback)); - } else { - _tests.pop_back(); - return 500; - } - - char json[512]; - Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\",\"timestamp\":%llu}",test->testId,test->timestamp); - responseBody = json; - responseContentType = "application/json"; - - return 200; - } // else 404 } else { @@ -1118,7 +1065,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( void EmbeddedNetworkController::threadMain() throw() { - uint64_t lastCircuitTestCheck = 0; _RQEntry *qe = (_RQEntry *)0; while ((_running)&&(_queue.get(qe))) { try { @@ -1153,80 +1099,9 @@ void EmbeddedNetworkController::threadMain() } } catch ( ... ) {} delete qe; - - if (_running) { - uint64_t now = OSUtils::now(); - if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - lastCircuitTestCheck = now; - Mutex::Lock _l(_tests_m); - for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { - if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - _node->circuitTestEnd(&(*i)); - _tests.erase(i++); - } else ++i; - } - } - } } } -void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) -{ - char tmp[2048],id[128]; - EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); - - if ((!test)||(!report)||(!test->credentialNetworkId)) return; // sanity check - - const uint64_t now = OSUtils::now(); - Utils::snprintf(id,sizeof(id),"network/%.16llx/test/%.16llx-%.16llx-%.10llx-%.10llx",test->credentialNetworkId,test->testId,now,report->upstream,report->current); - Utils::snprintf(tmp,sizeof(tmp), - "{\"id\": \"%s\"," - "\"objtype\": \"circuit_test\"," - "\"timestamp\": %llu," - "\"networkId\": \"%.16llx\"," - "\"testId\": \"%.16llx\"," - "\"upstream\": \"%.10llx\"," - "\"current\": \"%.10llx\"," - "\"receivedTimestamp\": %llu," - "\"sourcePacketId\": \"%.16llx\"," - "\"flags\": %llu," - "\"sourcePacketHopCount\": %u," - "\"errorCode\": %u," - "\"vendor\": %d," - "\"protocolVersion\": %u," - "\"majorVersion\": %u," - "\"minorVersion\": %u," - "\"revision\": %u," - "\"platform\": %d," - "\"architecture\": %d," - "\"receivedOnLocalAddress\": \"%s\"," - "\"receivedFromRemoteAddress\": \"%s\"," - "\"receivedFromLinkQuality\": %f}", - id + 30, // last bit only, not leading path - (unsigned long long)test->timestamp, - (unsigned long long)test->credentialNetworkId, - (unsigned long long)test->testId, - (unsigned long long)report->upstream, - (unsigned long long)report->current, - (unsigned long long)now, - (unsigned long long)report->sourcePacketId, - (unsigned long long)report->flags, - report->sourcePacketHopCount, - report->errorCode, - (int)report->vendor, - report->protocolVersion, - report->majorVersion, - report->minorVersion, - report->revision, - (int)report->platform, - (int)report->architecture, - reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str(), - ((double)report->receivedFromLinkQuality / (double)ZT_PATH_LINK_QUALITY_MAX)); - - self->_db.writeRaw(id,std::string(tmp)); -} - void EmbeddedNetworkController::_request( uint64_t nwid, const InetAddress &fromAddr, diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 4f4660f8..1589ea71 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -45,9 +45,6 @@ #include "JSONDB.hpp" -// TTL for circuit tests -#define ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION 120000 - namespace ZeroTier { class Node; @@ -110,7 +107,6 @@ private: } type; }; - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); inline void _startThreads() @@ -219,9 +215,6 @@ private: NetworkController::Sender *_sender; Identity _signingId; - std::list< ZT_CircuitTest > _tests; - Mutex _tests_m; - struct _MemberStatusKey { _MemberStatusKey() : networkId(0),nodeId(0) {} diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 17d6d67e..5126c5a2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -760,7 +760,6 @@ typedef struct */ uint64_t expiration; - struct { uint64_t from; uint64_t to; @@ -1105,197 +1104,6 @@ typedef struct unsigned long peerCount; } ZT_PeerList; -/** - * ZeroTier circuit test configuration and path - */ -typedef struct { - /** - * Test ID -- an arbitrary 64-bit identifier - */ - uint64_t testId; - - /** - * Timestamp -- sent with test and echoed back by each reporter - */ - uint64_t timestamp; - - /** - * Originator credential: network ID - * - * If this is nonzero, a network ID will be set for this test and - * the originator must be its primary network controller. This is - * currently the only authorization method available, so it must - * be set to run a test. - */ - uint64_t credentialNetworkId; - - /** - * Hops in circuit test (a.k.a. FIFO for graph traversal) - */ - struct { - /** - * Hop flags (currently unused, must be zero) - */ - unsigned int flags; - - /** - * Number of addresses in this hop (max: ZT_CIRCUIT_TEST_MAX_HOP_BREADTH) - */ - unsigned int breadth; - - /** - * 40-bit ZeroTier addresses (most significant 24 bits ignored) - */ - uint64_t addresses[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH]; - } hops[ZT_CIRCUIT_TEST_MAX_HOPS]; - - /** - * Number of hops (max: ZT_CIRCUIT_TEST_MAX_HOPS) - */ - unsigned int hopCount; - - /** - * If non-zero, circuit test will report back at every hop - */ - int reportAtEveryHop; - - /** - * An arbitrary user-settable pointer - */ - void *ptr; - - /** - * Pointer for internal use -- initialize to zero and do not modify - */ - void *_internalPtr; -} ZT_CircuitTest; - -/** - * Circuit test result report - */ -typedef struct { - /** - * Sender of report (current hop) - */ - uint64_t current; - - /** - * Previous hop - */ - uint64_t upstream; - - /** - * 64-bit test ID - */ - uint64_t testId; - - /** - * Timestamp from original test (echoed back at each hop) - */ - uint64_t timestamp; - - /** - * 64-bit packet ID of packet received by the reporting device - */ - uint64_t sourcePacketId; - - /** - * Flags - */ - uint64_t flags; - - /** - * ZeroTier protocol-level hop count of packet received by reporting device (>0 indicates relayed) - */ - unsigned int sourcePacketHopCount; - - /** - * Error code (currently unused, will be zero) - */ - unsigned int errorCode; - - /** - * Remote device vendor ID - */ - enum ZT_Vendor vendor; - - /** - * Remote device protocol compliance version - */ - unsigned int protocolVersion; - - /** - * Software major version - */ - unsigned int majorVersion; - - /** - * Software minor version - */ - unsigned int minorVersion; - - /** - * Software revision - */ - unsigned int revision; - - /** - * Platform / OS - */ - enum ZT_Platform platform; - - /** - * System architecture - */ - enum ZT_Architecture architecture; - - /** - * Local device address on which packet was received by reporting device - * - * This may have ss_family equal to zero (null address) if unspecified. - */ - struct sockaddr_storage receivedOnLocalAddress; - - /** - * Remote address from which reporter received the test packet - * - * This may have ss_family set to zero (null address) if unspecified. - */ - struct sockaddr_storage receivedFromRemoteAddress; - - /** - * Path link quality of physical path over which test was received - */ - int receivedFromLinkQuality; - - /** - * Next hops to which packets are being or will be sent by the reporter - * - * In addition to reporting back, the reporter may send the test on if - * there are more recipients in the FIFO. If it does this, it can report - * back the address(es) that make up the next hop and the physical address - * for each if it has one. The physical address being null/unspecified - * typically indicates that no direct path exists and the next packet - * will be relayed. - */ - struct { - /** - * 40-bit ZeroTier address - */ - uint64_t address; - - /** - * Physical address or null address (ss_family == 0) if unspecified or unknown - */ - struct sockaddr_storage physicalAddress; - } nextHops[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH]; - - /** - * Number of next hops reported in nextHops[] - */ - unsigned int nextHopCount; -} ZT_CircuitTestReport; - /** * A cluster member's status */ @@ -1957,40 +1765,6 @@ int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t type */ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); -/** - * Initiate a VL1 circuit test - * - * This sends an initial VERB_CIRCUIT_TEST and reports results back to the - * supplied callback until circuitTestEnd() is called. The supplied - * ZT_CircuitTest structure should be initially zeroed and then filled - * in with settings and hops. - * - * It is the caller's responsibility to call circuitTestEnd() and then - * to dispose of the test structure. Otherwise this node will listen - * for results forever. - * - * @param node Node instance - * @param tptr Thread pointer to pass to functions/callbacks resulting from this call - * @param test Test configuration - * @param reportCallback Function to call each time a report is received - * @return OK or error if, for example, test is too big for a packet or support isn't compiled in - */ -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *)); - -/** - * Stop listening for results to a given circuit test - * - * This does not free the 'test' structure. The caller may do that - * after calling this method to unregister it. - * - * Any reports that are received for a given test ID after it is - * terminated are ignored. - * - * @param node Node instance - * @param test Test configuration to unregister - */ -void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test); - /** * Initialize cluster operation * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 131659f9..9140c502 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -115,8 +115,6 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) case Packet::VERB_MULTICAST_GATHER: return _doMULTICAST_GATHER(RR,tPtr,peer); case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); - case Packet::VERB_CIRCUIT_TEST: return _doCIRCUIT_TEST(RR,tPtr,peer); - case Packet::VERB_CIRCUIT_TEST_REPORT: return _doCIRCUIT_TEST_REPORT(RR,tPtr,peer); case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); } } else { @@ -1252,196 +1250,6 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt return true; } -bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) -{ - try { - const Address originatorAddress(field(ZT_PACKET_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - SharedPtr originator(RR->topology->getPeer(tPtr,originatorAddress)); - if (!originator) { - RR->sw->requestWhois(tPtr,originatorAddress); - return false; - } - - const unsigned int flags = at(ZT_PACKET_IDX_PAYLOAD + 5); - const uint64_t timestamp = at(ZT_PACKET_IDX_PAYLOAD + 7); - const uint64_t testId = at(ZT_PACKET_IDX_PAYLOAD + 15); - - // Tracks total length of variable length fields, initialized to originator credential length below - unsigned int vlf; - - // Originator credentials -- right now only a network ID for which the originator is controller or is authorized by controller is allowed - const unsigned int originatorCredentialLength = vlf = at(ZT_PACKET_IDX_PAYLOAD + 23); - uint64_t originatorCredentialNetworkId = 0; - if (originatorCredentialLength >= 1) { - switch((*this)[ZT_PACKET_IDX_PAYLOAD + 25]) { - case 0x01: { // 64-bit network ID, originator must be controller - if (originatorCredentialLength >= 9) - originatorCredentialNetworkId = at(ZT_PACKET_IDX_PAYLOAD + 26); - } break; - default: break; - } - } - - // Add length of "additional fields," which are currently unused - vlf += at(ZT_PACKET_IDX_PAYLOAD + 25 + vlf); - - // Verify signature -- only tests signed by their originators are allowed - const unsigned int signatureLength = at(ZT_PACKET_IDX_PAYLOAD + 27 + vlf); - if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) { - TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - return true; - } - vlf += signatureLength; - - // Save this length so we can copy the immutable parts of this test - // into the one we send along to next hops. - const unsigned int lengthOfSignedPortionAndSignature = 29 + vlf; - - // Add length of second "additional fields" section. - vlf += at(ZT_PACKET_IDX_PAYLOAD + 29 + vlf); - - uint64_t reportFlags = 0; - - // Check credentials (signature already verified) - if (originatorCredentialNetworkId) { - SharedPtr network(RR->node->network(originatorCredentialNetworkId)); - if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - return true; - } - if (network->gate(tPtr,peer)) - reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH; - } else { - TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - return true; - } - - const uint64_t now = RR->node->now(); - - unsigned int breadth = 0; - Address nextHop[256]; // breadth is a uin8_t, so this is the max - InetAddress nextHopBestPathAddress[256]; - unsigned int remainingHopsPtr = ZT_PACKET_IDX_PAYLOAD + 33 + vlf; - if ((ZT_PACKET_IDX_PAYLOAD + 31 + vlf) < size()) { - // unsigned int nextHopFlags = (*this)[ZT_PACKET_IDX_PAYLOAD + 31 + vlf] - breadth = (*this)[ZT_PACKET_IDX_PAYLOAD + 32 + vlf]; - for(unsigned int h=0;h nhp(RR->topology->getPeer(tPtr,nextHop[h])); - if (nhp) { - SharedPtr nhbp(nhp->getBestPath(now,false)); - if ((nhbp)&&(nhbp->alive(now))) - nextHopBestPathAddress[h] = nhbp->address(); - } - } - } - - // Report back to originator, depending on flags and whether we are last hop - if ( ((flags & 0x01) != 0) || ((breadth == 0)&&((flags & 0x02) != 0)) ) { - Packet outp(originatorAddress,RR->identity.address(),Packet::VERB_CIRCUIT_TEST_REPORT); - outp.append((uint64_t)timestamp); - outp.append((uint64_t)testId); - outp.append((uint64_t)0); // field reserved for future use - outp.append((uint8_t)ZT_VENDOR_ZEROTIER); - outp.append((uint8_t)ZT_PROTO_VERSION); - outp.append((uint8_t)ZEROTIER_ONE_VERSION_MAJOR); - outp.append((uint8_t)ZEROTIER_ONE_VERSION_MINOR); - outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); - outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED); - outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED); - outp.append((uint16_t)0); // error code, currently unused - outp.append((uint64_t)reportFlags); - outp.append((uint64_t)packetId()); - peer->address().appendTo(outp); - outp.append((uint8_t)hops()); - _path->localAddress().serialize(outp); - _path->address().serialize(outp); - outp.append((uint16_t)_path->linkQuality()); - outp.append((uint8_t)breadth); - for(unsigned int h=0;hsw->send(tPtr,outp,true); - } - - // If there are next hops, forward the test along through the graph - if (breadth > 0) { - Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST); - outp.append(field(ZT_PACKET_IDX_PAYLOAD,lengthOfSignedPortionAndSignature),lengthOfSignedPortionAndSignature); - outp.append((uint16_t)0); // no additional fields - if (remainingHopsPtr < size()) - outp.append(field(remainingHopsPtr,size() - remainingHopsPtr),size() - remainingHopsPtr); - - for(unsigned int h=0;hidentity.address() != nextHop[h]) { // next hops that loop back to the current hop are not valid - outp.newInitializationVector(); - outp.setDestination(nextHop[h]); - RR->sw->send(tPtr,outp,true); - } - } - } - - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false); - } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); - } - return true; -} - -bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) -{ - try { - ZT_CircuitTestReport report; - memset(&report,0,sizeof(report)); - - report.current = peer->address().toInt(); - report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); - report.testId = at(ZT_PACKET_IDX_PAYLOAD + 8); - report.timestamp = at(ZT_PACKET_IDX_PAYLOAD); - report.sourcePacketId = at(ZT_PACKET_IDX_PAYLOAD + 44); - report.flags = at(ZT_PACKET_IDX_PAYLOAD + 36); - report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58 - report.errorCode = at(ZT_PACKET_IDX_PAYLOAD + 34); - report.vendor = (enum ZT_Vendor)((*this)[ZT_PACKET_IDX_PAYLOAD + 24]); - report.protocolVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 25]; - report.majorVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 26]; - report.minorVersion = (*this)[ZT_PACKET_IDX_PAYLOAD + 27]; - report.revision = at(ZT_PACKET_IDX_PAYLOAD + 28); - report.platform = (enum ZT_Platform)at(ZT_PACKET_IDX_PAYLOAD + 30); - report.architecture = (enum ZT_Architecture)at(ZT_PACKET_IDX_PAYLOAD + 32); - - const unsigned int receivedOnLocalAddressLen = reinterpret_cast(&(report.receivedOnLocalAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58); - const unsigned int receivedFromRemoteAddressLen = reinterpret_cast(&(report.receivedFromRemoteAddress))->deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen); - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 58 + receivedOnLocalAddressLen + receivedFromRemoteAddressLen; - if (report.protocolVersion >= 9) { - report.receivedFromLinkQuality = at(ptr); ptr += 2; - } else { - report.receivedFromLinkQuality = ZT_PATH_LINK_QUALITY_MAX; - ptr += at(ptr) + 2; // this field was once an 'extended field length' reserved field, which was always set to 0 - } - - report.nextHopCount = (*this)[ptr++]; - if (report.nextHopCount > ZT_CIRCUIT_TEST_MAX_HOP_BREADTH) // sanity check, shouldn't be possible - report.nextHopCount = ZT_CIRCUIT_TEST_MAX_HOP_BREADTH; - for(unsigned int h=0;h(&(report.nextHops[h].physicalAddress))->deserialize(*this,ptr); - } - - RR->node->postCircuitTestReport(&report); - - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); - } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); - } - return true; -} - bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { @@ -1453,9 +1261,9 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + TRACE("dropped USER_MESSAGE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); } return true; } diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 43a1ea10..11b60712 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -138,8 +138,6 @@ private: bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); - bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); - bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid); diff --git a/node/Node.cpp b/node/Node.cpp index 6d7eea43..911c9c4b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -503,64 +503,6 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } -ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) -{ - if (test->hopCount > 0) { - try { - Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST); - RR->identity.address().appendTo(outp); - outp.append((uint16_t)((test->reportAtEveryHop != 0) ? 0x03 : 0x02)); - outp.append((uint64_t)test->timestamp); - outp.append((uint64_t)test->testId); - outp.append((uint16_t)0); // originator credential length, updated later - if (test->credentialNetworkId) { - outp.append((uint8_t)0x01); - outp.append((uint64_t)test->credentialNetworkId); - outp.setAt(ZT_PACKET_IDX_PAYLOAD + 23,(uint16_t)9); - } - outp.append((uint16_t)0); - C25519::Signature sig(RR->identity.sign(reinterpret_cast(outp.data()) + ZT_PACKET_IDX_PAYLOAD,outp.size() - ZT_PACKET_IDX_PAYLOAD)); - outp.append((uint16_t)sig.size()); - outp.append(sig.data,(unsigned int)sig.size()); - outp.append((uint16_t)0); // originator doesn't need an extra credential, since it's the originator - for(unsigned int h=1;hhopCount;++h) { - outp.append((uint8_t)0); - outp.append((uint8_t)(test->hops[h].breadth & 0xff)); - for(unsigned int a=0;ahops[h].breadth;++a) - Address(test->hops[h].addresses[a]).appendTo(outp); - } - - for(unsigned int a=0;ahops[0].breadth;++a) { - outp.newInitializationVector(); - outp.setDestination(Address(test->hops[0].addresses[a])); - RR->sw->send(tptr,outp,true); - } - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet - } - } - - { - test->_internalPtr = reinterpret_cast(reportCallback); - Mutex::Lock _l(_circuitTests_m); - if (std::find(_circuitTests.begin(),_circuitTests.end(),test) == _circuitTests.end()) - _circuitTests.push_back(test); - } - - return ZT_RESULT_OK; -} - -void Node::circuitTestEnd(ZT_CircuitTest *test) -{ - Mutex::Lock _l(_circuitTests_m); - for(;;) { - std::vector< ZT_CircuitTest * >::iterator ct(std::find(_circuitTests.begin(),_circuitTests.end(),test)); - if (ct == _circuitTests.end()) - break; - else _circuitTests.erase(ct); - } -} - ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -715,20 +657,6 @@ uint64_t Node::prng() return z + y; } -void Node::postCircuitTestReport(const ZT_CircuitTestReport *report) -{ - std::vector< ZT_CircuitTest * > toNotify; - { - Mutex::Lock _l(_circuitTests_m); - for(std::vector< ZT_CircuitTest * >::iterator i(_circuitTests.begin());i!=_circuitTests.end();++i) { - if ((*i)->testId == report->testId) - toNotify.push_back(*i); - } - } - for(std::vector< ZT_CircuitTest * >::iterator i(toNotify.begin());i!=toNotify.end();++i) - (reinterpret_cast((*i)->_internalPtr))(reinterpret_cast(this),*i,report); -} - void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count) { RR->topology->setTrustedPaths(reinterpret_cast(networks),ids,count); @@ -1070,22 +998,6 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) -{ - try { - return reinterpret_cast(node)->circuitTestBegin(tptr,test,reportCallback); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - -void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test) -{ - try { - reinterpret_cast(node)->circuitTestEnd(test); - } catch ( ... ) {} -} - enum ZT_ResultCode ZT_Node_clusterInit( ZT_Node *node, unsigned int myId, diff --git a/node/Node.hpp b/node/Node.hpp index 95587161..57b5489e 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -117,8 +117,6 @@ public: void clearLocalInterfaceAddresses(); int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setNetconfMaster(void *networkControllerInstance); - ZT_ResultCode circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)); - void circuitTestEnd(ZT_CircuitTest *test); ZT_ResultCode clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -219,7 +217,6 @@ public: inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); - void postCircuitTestReport(const ZT_CircuitTestReport *report); void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); World planet() const; @@ -309,9 +306,6 @@ private: std::vector< std::pair< uint64_t, SharedPtr > > _networks; Mutex _networks_m; - std::vector< ZT_CircuitTest * > _circuitTests; - Mutex _circuitTests_m; - std::vector _directPaths; Mutex _directPaths_m; diff --git a/node/Packet.cpp b/node/Packet.cpp index e778e3bb..6e1b36ac 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1082,8 +1082,6 @@ const char *Packet::verbString(Verb v) case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; - case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST"; - case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT"; case VERB_USER_MESSAGE: return "USER_MESSAGE"; } return "(unknown)"; diff --git a/node/Packet.hpp b/node/Packet.hpp index 1de679e7..a76d4180 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -61,7 +61,7 @@ * 4 - 0.6.0 ... 1.0.6 * + BREAKING CHANGE: New identity format based on hashcash design * 5 - 1.1.0 ... 1.1.5 - * + Supports circuit test, proof of work, and echo + * + Supports echo * + Supports in-band world (root server definition) updates * + Clustering! (Though this will work with protocol v4 clients.) * + Otherwise backward compatible with protocol v4 @@ -954,119 +954,7 @@ public: */ VERB_PUSH_DIRECT_PATHS = 0x10, - /** - * Source-routed circuit test message: - * <[5] address of originator of circuit test> - * <[2] 16-bit flags> - * <[8] 64-bit timestamp> - * <[8] 64-bit test ID (arbitrary, set by tester)> - * <[2] 16-bit originator credential length (includes type)> - * [[1] originator credential type (for authorizing test)] - * [[...] originator credential] - * <[2] 16-bit length of additional fields> - * [[...] additional fields] - * [ ... end of signed portion of request ... ] - * <[2] 16-bit length of signature of request> - * <[...] signature of request by originator> - * <[2] 16-bit length of additional fields> - * [[...] additional fields] - * <[...] next hop(s) in path> - * - * Flags: - * 0x01 - Report back to originator at all hops - * 0x02 - Report back to originator at last hop - * - * Originator credential types: - * 0x01 - 64-bit network ID for which originator is controller - * - * Path record format: - * <[1] 8-bit flags (unused, must be zero)> - * <[1] 8-bit breadth (number of next hops)> - * <[...] one or more ZeroTier addresses of next hops> - * - * The circuit test allows a device to send a message that will traverse - * the network along a specified path, with each hop optionally reporting - * back to the tester via VERB_CIRCUIT_TEST_REPORT. - * - * Each circuit test packet includes a digital signature by the originator - * of the request, as well as a credential by which that originator claims - * authorization to perform the test. Currently this signature is ed25519, - * but in the future flags might be used to indicate an alternative - * algorithm. For example, the originator might be a network controller. - * In this case the test might be authorized if the recipient is a member - * of a network controlled by it, and if the previous hop(s) are also - * members. Each hop may include its certificate of network membership. - * - * Circuit test paths consist of a series of records. When a node receives - * an authorized circuit test, it: - * - * (1) Reports back to circuit tester as flags indicate - * (2) Reads and removes the next hop from the packet's path - * (3) Sends the packet along to next hop(s), if any. - * - * It is perfectly legal for a path to contain the same hop more than - * once. In fact, this can be a very useful test to determine if a hop - * can be reached bidirectionally and if so what that connectivity looks - * like. - * - * The breadth field in source-routed path records allows a hop to forward - * to more than one recipient, allowing the tester to specify different - * forms of graph traversal in a test. - * - * There is no hard limit to the number of hops in a test, but it is - * practically limited by the maximum size of a (possibly fragmented) - * ZeroTier packet. - * - * Support for circuit tests is optional. If they are not supported, the - * node should respond with an UNSUPPORTED_OPERATION error. If a circuit - * test request is not authorized, it may be ignored or reported as - * an INVALID_REQUEST. No OK messages are generated, but TEST_REPORT - * messages may be sent (see below). - * - * ERROR packet format: - * <[8] 64-bit timestamp (echoed from original> - * <[8] 64-bit test ID (echoed from original)> - */ - VERB_CIRCUIT_TEST = 0x11, - - /** - * Circuit test hop report: - * <[8] 64-bit timestamp (echoed from original test)> - * <[8] 64-bit test ID (echoed from original test)> - * <[8] 64-bit reserved field (set to 0, currently unused)> - * <[1] 8-bit vendor ID (set to 0, currently unused)> - * <[1] 8-bit reporter protocol version> - * <[1] 8-bit reporter software major version> - * <[1] 8-bit reporter software minor version> - * <[2] 16-bit reporter software revision> - * <[2] 16-bit reporter OS/platform or 0 if not specified> - * <[2] 16-bit reporter architecture or 0 if not specified> - * <[2] 16-bit error code (set to 0, currently unused)> - * <[8] 64-bit report flags> - * <[8] 64-bit packet ID of received CIRCUIT_TEST packet> - * <[5] upstream ZeroTier address from which CIRCUIT_TEST was received> - * <[1] 8-bit packet hop count of received CIRCUIT_TEST> - * <[...] local wire address on which packet was received> - * <[...] remote wire address from which packet was received> - * <[2] 16-bit path link quality of path over which packet was received> - * <[1] 8-bit number of next hops (breadth)> - * <[...] next hop information> - * - * Next hop information record format: - * <[5] ZeroTier address of next hop> - * <[...] current best direct path address, if any, 0 if none> - * - * Report flags: - * 0x1 - Upstream peer in circuit test path allowed in path (e.g. network COM valid) - * - * Circuit test reports can be sent by hops in a circuit test to report - * back results. They should include information about the sender as well - * as about the paths to which next hops are being sent. - * - * If a test report is received and no circuit test was sent, it should be - * ignored. This message generates no OK or ERROR response. - */ - VERB_CIRCUIT_TEST_REPORT = 0x12, + // 0x11, 0x12 -- deprecated /** * A message with arbitrary user-definable content: -- cgit v1.2.3 From 2a4a50b1daaec74d7a4d08869ead31ff1f966fa1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 May 2017 08:36:09 -0700 Subject: Add some also-ZeroTier-written ext/ code for use in new clustering, delete some old code, and change Mac to use -Os which is just as fast as -Ofast and may be faster due to cache effects. --- ext/kissdb/Makefile | 7 + ext/kissdb/README.md | 69 +++ ext/kissdb/SPEC.txt | 62 ++ ext/kissdb/kissdb.c | 452 +++++++++++++++ ext/kissdb/kissdb.h | 173 ++++++ ext/vsdm/LICENSE.txt | 22 + ext/vsdm/Makefile | 5 + ext/vsdm/README.md | 40 ++ ext/vsdm/vsdm-test.cpp | 72 +++ ext/vsdm/vsdm.hpp | 1491 ++++++++++++++++++++++++++++++++++++++++++++++++ include/ZeroTierOne.h | 197 ------- make-mac.mk | 2 +- node/Node.cpp | 52 +- node/Node.hpp | 15 - 14 files changed, 2396 insertions(+), 263 deletions(-) create mode 100644 ext/kissdb/Makefile create mode 100644 ext/kissdb/README.md create mode 100644 ext/kissdb/SPEC.txt create mode 100644 ext/kissdb/kissdb.c create mode 100644 ext/kissdb/kissdb.h create mode 100644 ext/vsdm/LICENSE.txt create mode 100644 ext/vsdm/Makefile create mode 100644 ext/vsdm/README.md create mode 100644 ext/vsdm/vsdm-test.cpp create mode 100644 ext/vsdm/vsdm.hpp (limited to 'node') diff --git a/ext/kissdb/Makefile b/ext/kissdb/Makefile new file mode 100644 index 00000000..f47372c6 --- /dev/null +++ b/ext/kissdb/Makefile @@ -0,0 +1,7 @@ +# http://creativecommons.org/publicdomain/zero/1.0/ + +all: + gcc -Wall -O2 -DKISSDB_TEST -o kissdb-test kissdb.c + +clean: + rm -f kissdb-test *.o test.db diff --git a/ext/kissdb/README.md b/ext/kissdb/README.md new file mode 100644 index 00000000..ab8a6cff --- /dev/null +++ b/ext/kissdb/README.md @@ -0,0 +1,69 @@ +kissdb +====== + +(Keep It) Simple Stupid Database + +KISSDB is about the simplest key/value store you'll ever see, anywhere. +It's written in plain vanilla C using only the standard string and FILE +I/O functions, and should port to just about anything with a disk or +something that acts like one. + +It stores keys and values of fixed length in a stupid-simple file format +based on fixed-size hash tables. If a hash collision occurrs, a new "page" +of hash table is appended to the database. The format is append-only. +There is no delete. Puts that replace an existing value, however, will not +grow the file as they will overwrite the existing entry. + +Hash table size is a space/speed trade-off parameter. Larger hash tables +will reduce collisions and speed things up a bit, at the expense of memory +and disk space. A good size is usually about 1/2 the average number of +entries you expect. + +Features: + + * Tiny, compiles to ~4k on an x86_64 Linux system + * Small memory footprint (only caches hash tables) + * Very space-efficient (on disk) if small hash tables are used + * Makes a decent effort to be robust on power loss + * Pretty respectably fast, especially given its simplicity + * 64-bit, file size limit is 2^64 bytes + * Ports to anything with a C compiler and stdlib/stdio + * Public domain + +Limitations: + + * Fixed-size keys and values, must recreate and copy to change any init size parameter + * Add/update only, no delete + * Iteration is supported but key order is undefined + * No search for subsets of keys/values + * No indexes + * No transactions + * No special recovery features if a database gets corrupted + * No built-in thread-safety (guard it with a mutex in MT code) + * No built-in caching of data (only hash tables are cached for lookup speed) + * No endian-awareness (currently), so big-endian DBs won't read on little-endian machines + +Alternative key/value stores and embedded databases: + + * [MDB](http://symas.com/mdb/) uses mmap() and is very fast (not quite as tiny/simple/portable) + * [CDB](http://cr.yp.to/cdb.html) is also minimal and fast, probably the closest thing to this (but has a 4gb size limit) + * [Kyoto Cabinet](http://fallabs.com/kyotocabinet/) is very fast, full-featured, and modern (license required for commercial use) + * [SQLite](http://www.sqlite.org/) gives you a complete embedded SQL server (public domain, very mature, much larger) + * Others include GDBM, NDBM, Berkeley DB, etc. Use your Googles. :) + +KISSDB is good if you want space-efficient relatively fast write-once/read-many storage +of keys mapped to values. It's not a good choice if you need searches, indexes, delete, +structured storage, or widely varying key/value sizes. It's also probably not a good +choice if you need a long-lived database for critical data, since it lacks recovery +features and is brittle if its internals are modified. It would be better for a cache +of data that can be restored or "re-learned," such as keys, Bitcoin transactions, nodes +on a peer-to-peer network, log analysis results, rendered web pages, session cookies, +auth tokens, etc. + +KISSDB is in the public domain as according to the [Creative Commons Public Domain Dedication](http://creativecommons.org/publicdomain/zero/1.0/). +One reason it was written was the poverty of simple key/value databases with wide open licensing. Even old ones like GDBM have GPL, not LGPL, licenses. + +See comments in kissdb.h for documentation. Makefile can be used to build +a test program on systems with gcc. + +Author: Adam Ierymenko / ZeroTier Networks LLC diff --git a/ext/kissdb/SPEC.txt b/ext/kissdb/SPEC.txt new file mode 100644 index 00000000..732c4df5 --- /dev/null +++ b/ext/kissdb/SPEC.txt @@ -0,0 +1,62 @@ +----- + +KISSDB file format (version 2) +Author: Adam Ierymenko + +http://creativecommons.org/publicdomain/zero/1.0/ + +----- + +In keeping with the goal of minimalism the file format is very simple, the +sort of thing that would be given as an example in an introductory course in +data structures. It's a basic hash table that adds additional pages of hash +table entries on collision. + +It consists of a 28 byte header followed by a series of hash tables and data. +All integer values are stored in the native word order of the target +architecture (in the future the code might be fixed to make everything +little-endian if anyone cares about that). + +The header consists of the following fields: + +[0-3] magic numbers: (ASCII) 'K', 'd', 'B', KISSDB_VERSION (currently 2) +[4-11] 64-bit hash table size in entries +[12-19] 64-bit key size in bytes +[20-27] 64-bit value size in bytes + +Hash tables are arrays of [hash table size + 1] 64-bit integers. The extra +entry, if nonzero, is the offset in the file of the next hash table, forming +a linked list of hash tables across the file. + +Immediately following the header, the first hash table will be written when +the first key/value is added. The algorithm for adding new entries is as +follows: + +(1) The key is hashed using a 64-bit variant of the DJB2 hash function, and + this is taken modulo hash table size to get a bucket number. +(2) Hash tables are checked in order, starting with the first hash table, + until a zero (empty) bucket is found. If one is found, skip to step (4). +(3) If no empty buckets are found in any hash table, a new table is appended + to the file and the final pointer in the previous hash table is set to + its offset. (In the code the update of the next hash table pointer in + the previous hash table happens last, after the whole write is complete, + to avoid corruption on power loss.) +(4) The key and value are appended, in order with no additional meta-data, + to the database file. Before appending the offset in the file stream + where they will be stored is saved. After appending, this offset is + written to the empty hash table bucket we chose in steps 2/3. Hash table + updates happen last to avoid corruption if the write does not complete. + +Lookup of a key/value pair occurs as follows: + +(1) The key is hashed and taken modulo hash table size to get a bucket + number. +(2) If this bucket's entry in the hash table is nonzero, the key at the + offset specified by this bucket is compared to the key being looked up. + If they are equal, the value is read and returned. +(3) If the keys are not equal, the next hash table is checked and step (2) + is repeated. If an empty bucket is encountered or if we run out of hash + tables, the key was not found. + +To update an existing value, its location is looked up and the value portion +of the entry is rewritten. diff --git a/ext/kissdb/kissdb.c b/ext/kissdb/kissdb.c new file mode 100644 index 00000000..6b275686 --- /dev/null +++ b/ext/kissdb/kissdb.c @@ -0,0 +1,452 @@ +/* (Keep It) Simple Stupid Database + * + * Written by Adam Ierymenko + * KISSDB is in the public domain and is distributed with NO WARRANTY. + * + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Compile with KISSDB_TEST to build as a test program. */ + +/* Note: big-endian systems will need changes to implement byte swapping + * on hash table file I/O. Or you could just use it as-is if you don't care + * that your database files will be unreadable on little-endian systems. */ + +#define _FILE_OFFSET_BITS 64 + +#include "kissdb.h" + +#include +#include +#include + +#ifdef _WIN32 +#define fseeko _fseeki64 +#define ftello _ftelli64 +#endif + +#define KISSDB_HEADER_SIZE ((sizeof(uint64_t) * 3) + 4) + +/* djb2 hash function */ +static uint64_t KISSDB_hash(const void *b,unsigned long len) +{ + unsigned long i; + uint64_t hash = 5381; + for(i=0;if = (FILE *)0; + fopen_s(&db->f,path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); +#else + db->f = fopen(path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); +#endif + if (!db->f) { + if (mode == KISSDB_OPEN_MODE_RWCREAT) { +#ifdef _WIN32 + db->f = (FILE *)0; + fopen_s(&db->f,path,"w+b"); +#else + db->f = fopen(path,"w+b"); +#endif + } + if (!db->f) + return KISSDB_ERROR_IO; + } + + if (fseeko(db->f,0,SEEK_END)) { + fclose(db->f); + return KISSDB_ERROR_IO; + } + if (ftello(db->f) < KISSDB_HEADER_SIZE) { + /* write header if not already present */ + if ((hash_table_size)&&(key_size)&&(value_size)) { + if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp2[0] = 'K'; tmp2[1] = 'd'; tmp2[2] = 'B'; tmp2[3] = KISSDB_VERSION; + if (fwrite(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp = hash_table_size; + if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp = key_size; + if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp = value_size; + if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + fflush(db->f); + } else { + fclose(db->f); + return KISSDB_ERROR_INVALID_PARAMETERS; + } + } else { + if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } + if (fread(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if ((tmp2[0] != 'K')||(tmp2[1] != 'd')||(tmp2[2] != 'B')||(tmp2[3] != KISSDB_VERSION)) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if (!tmp) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + hash_table_size = (unsigned long)tmp; + if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if (!tmp) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + key_size = (unsigned long)tmp; + if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if (!tmp) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + value_size = (unsigned long)tmp; + } + + db->hash_table_size = hash_table_size; + db->key_size = key_size; + db->value_size = value_size; + db->hash_table_size_bytes = sizeof(uint64_t) * (hash_table_size + 1); /* [hash_table_size] == next table */ + + httmp = malloc(db->hash_table_size_bytes); + if (!httmp) { + fclose(db->f); + return KISSDB_ERROR_MALLOC; + } + db->num_hash_tables = 0; + db->hash_tables = (uint64_t *)0; + while (fread(httmp,db->hash_table_size_bytes,1,db->f) == 1) { + hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); + if (!hash_tables_rea) { + KISSDB_close(db); + free(httmp); + return KISSDB_ERROR_MALLOC; + } + db->hash_tables = hash_tables_rea; + + memcpy(((uint8_t *)db->hash_tables) + (db->hash_table_size_bytes * db->num_hash_tables),httmp,db->hash_table_size_bytes); + ++db->num_hash_tables; + if (httmp[db->hash_table_size]) { + if (fseeko(db->f,httmp[db->hash_table_size],SEEK_SET)) { + KISSDB_close(db); + free(httmp); + return KISSDB_ERROR_IO; + } + } else break; + } + free(httmp); + + return 0; +} + +void KISSDB_close(KISSDB *db) +{ + if (db->hash_tables) + free(db->hash_tables); + if (db->f) + fclose(db->f); + memset(db,0,sizeof(KISSDB)); +} + +int KISSDB_get(KISSDB *db,const void *key,void *vbuf) +{ + uint8_t tmp[4096]; + const uint8_t *kptr; + unsigned long klen,i; + uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; + uint64_t offset; + uint64_t *cur_hash_table; + long n; + + cur_hash_table = db->hash_tables; + for(i=0;inum_hash_tables;++i) { + offset = cur_hash_table[hash]; + if (offset) { + if (fseeko(db->f,offset,SEEK_SET)) + return KISSDB_ERROR_IO; + + kptr = (const uint8_t *)key; + klen = db->key_size; + while (klen) { + n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); + if (n > 0) { + if (memcmp(kptr,tmp,n)) + goto get_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long)n; + } else return 1; /* not found */ + } + + if (fread(vbuf,db->value_size,1,db->f) == 1) + return 0; /* success */ + else return KISSDB_ERROR_IO; + } else return 1; /* not found */ +get_no_match_next_hash_table: + cur_hash_table += db->hash_table_size + 1; + } + + return 1; /* not found */ +} + +int KISSDB_put(KISSDB *db,const void *key,const void *value) +{ + uint8_t tmp[4096]; + const uint8_t *kptr; + unsigned long klen,i; + uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; + uint64_t offset; + uint64_t htoffset,lasthtoffset; + uint64_t endoffset; + uint64_t *cur_hash_table; + uint64_t *hash_tables_rea; + long n; + + lasthtoffset = htoffset = KISSDB_HEADER_SIZE; + cur_hash_table = db->hash_tables; + for(i=0;inum_hash_tables;++i) { + offset = cur_hash_table[hash]; + if (offset) { + /* rewrite if already exists */ + if (fseeko(db->f,offset,SEEK_SET)) + return KISSDB_ERROR_IO; + + kptr = (const uint8_t *)key; + klen = db->key_size; + while (klen) { + n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); + if (n > 0) { + if (memcmp(kptr,tmp,n)) + goto put_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long)n; + } + } + + /* C99 spec demands seek after fread(), required for Windows */ + fseeko(db->f,0,SEEK_CUR); + + if (fwrite(value,db->value_size,1,db->f) == 1) { + fflush(db->f); + return 0; /* success */ + } else return KISSDB_ERROR_IO; + } else { + /* add if an empty hash table slot is discovered */ + if (fseeko(db->f,0,SEEK_END)) + return KISSDB_ERROR_IO; + endoffset = ftello(db->f); + + if (fwrite(key,db->key_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + if (fwrite(value,db->value_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + + if (fseeko(db->f,htoffset + (sizeof(uint64_t) * hash),SEEK_SET)) + return KISSDB_ERROR_IO; + if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) + return KISSDB_ERROR_IO; + cur_hash_table[hash] = endoffset; + + fflush(db->f); + + return 0; /* success */ + } +put_no_match_next_hash_table: + lasthtoffset = htoffset; + htoffset = cur_hash_table[db->hash_table_size]; + cur_hash_table += (db->hash_table_size + 1); + } + + /* if no existing slots, add a new page of hash table entries */ + if (fseeko(db->f,0,SEEK_END)) + return KISSDB_ERROR_IO; + endoffset = ftello(db->f); + + hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); + if (!hash_tables_rea) + return KISSDB_ERROR_MALLOC; + db->hash_tables = hash_tables_rea; + cur_hash_table = &(db->hash_tables[(db->hash_table_size + 1) * db->num_hash_tables]); + memset(cur_hash_table,0,db->hash_table_size_bytes); + + cur_hash_table[hash] = endoffset + db->hash_table_size_bytes; /* where new entry will go */ + + if (fwrite(cur_hash_table,db->hash_table_size_bytes,1,db->f) != 1) + return KISSDB_ERROR_IO; + + if (fwrite(key,db->key_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + if (fwrite(value,db->value_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + + if (db->num_hash_tables) { + if (fseeko(db->f,lasthtoffset + (sizeof(uint64_t) * db->hash_table_size),SEEK_SET)) + return KISSDB_ERROR_IO; + if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) + return KISSDB_ERROR_IO; + db->hash_tables[((db->hash_table_size + 1) * (db->num_hash_tables - 1)) + db->hash_table_size] = endoffset; + } + + ++db->num_hash_tables; + + fflush(db->f); + + return 0; /* success */ +} + +void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi) +{ + dbi->db = db; + dbi->h_no = 0; + dbi->h_idx = 0; +} + +int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf) +{ + uint64_t offset; + + if ((dbi->h_no < dbi->db->num_hash_tables)&&(dbi->h_idx < dbi->db->hash_table_size)) { + while (!(offset = dbi->db->hash_tables[((dbi->db->hash_table_size + 1) * dbi->h_no) + dbi->h_idx])) { + if (++dbi->h_idx >= dbi->db->hash_table_size) { + dbi->h_idx = 0; + if (++dbi->h_no >= dbi->db->num_hash_tables) + return 0; + } + } + if (fseeko(dbi->db->f,offset,SEEK_SET)) + return KISSDB_ERROR_IO; + if (fread(kbuf,dbi->db->key_size,1,dbi->db->f) != 1) + return KISSDB_ERROR_IO; + if (fread(vbuf,dbi->db->value_size,1,dbi->db->f) != 1) + return KISSDB_ERROR_IO; + if (++dbi->h_idx >= dbi->db->hash_table_size) { + dbi->h_idx = 0; + ++dbi->h_no; + } + return 1; + } + + return 0; +} + +#ifdef KISSDB_TEST + +#include + +int main(int argc,char **argv) +{ + uint64_t i,j; + uint64_t v[8]; + KISSDB db; + KISSDB_Iterator dbi; + char got_all_values[10000]; + int q; + + printf("Opening new empty database test.db...\n"); + + if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RWREPLACE,1024,8,sizeof(v))) { + printf("KISSDB_open failed\n"); + return 1; + } + + printf("Adding and then re-getting 10000 64-byte values...\n"); + + for(i=0;i<10000;++i) { + for(j=0;j<8;++j) + v[j] = i; + if (KISSDB_put(&db,&i,v)) { + printf("KISSDB_put failed (%"PRIu64")\n",i); + return 1; + } + memset(v,0,sizeof(v)); + if ((q = KISSDB_get(&db,&i,v))) { + printf("KISSDB_get (1) failed (%"PRIu64") (%d)\n",i,q); + return 1; + } + for(j=0;j<8;++j) { + if (v[j] != i) { + printf("KISSDB_get (1) failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + } + + printf("Getting 10000 64-byte values...\n"); + + for(i=0;i<10000;++i) { + if ((q = KISSDB_get(&db,&i,v))) { + printf("KISSDB_get (2) failed (%"PRIu64") (%d)\n",i,q); + return 1; + } + for(j=0;j<8;++j) { + if (v[j] != i) { + printf("KISSDB_get (2) failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + } + + printf("Closing and re-opening database in read-only mode...\n"); + + KISSDB_close(&db); + + if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RDONLY,1024,8,sizeof(v))) { + printf("KISSDB_open failed\n"); + return 1; + } + + printf("Getting 10000 64-byte values...\n"); + + for(i=0;i<10000;++i) { + if ((q = KISSDB_get(&db,&i,v))) { + printf("KISSDB_get (3) failed (%"PRIu64") (%d)\n",i,q); + return 1; + } + for(j=0;j<8;++j) { + if (v[j] != i) { + printf("KISSDB_get (3) failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + } + + printf("Iterator test...\n"); + + KISSDB_Iterator_init(&db,&dbi); + i = 0xdeadbeef; + memset(got_all_values,0,sizeof(got_all_values)); + while (KISSDB_Iterator_next(&dbi,&i,&v) > 0) { + if (i < 10000) + got_all_values[i] = 1; + else { + printf("KISSDB_Iterator_next failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + for(i=0;i<10000;++i) { + if (!got_all_values[i]) { + printf("KISSDB_Iterator failed, missing value index %"PRIu64"\n",i); + return 1; + } + } + + KISSDB_close(&db); + + printf("All tests OK!\n"); + + return 0; +} + +#endif diff --git a/ext/kissdb/kissdb.h b/ext/kissdb/kissdb.h new file mode 100644 index 00000000..926906b0 --- /dev/null +++ b/ext/kissdb/kissdb.h @@ -0,0 +1,173 @@ +/* (Keep It) Simple Stupid Database + * + * Written by Adam Ierymenko + * KISSDB is in the public domain and is distributed with NO WARRANTY. + * + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +#ifndef ___KISSDB_H +#define ___KISSDB_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Version: 2 + * + * This is the file format identifier, and changes any time the file + * format changes. The code version will be this dot something, and can + * be seen in tags in the git repository. + */ +#define KISSDB_VERSION 2 + +/** + * KISSDB database state + * + * These fields can be read by a user, e.g. to look up key_size and + * value_size, but should never be changed. + */ +typedef struct { + unsigned long hash_table_size; + unsigned long key_size; + unsigned long value_size; + unsigned long hash_table_size_bytes; + unsigned long num_hash_tables; + uint64_t *hash_tables; + FILE *f; +} KISSDB; + +/** + * I/O error or file not found + */ +#define KISSDB_ERROR_IO -1 + +/** + * Out of memory + */ +#define KISSDB_ERROR_MALLOC -2 + +/** + * Invalid paramters (e.g. missing _size paramters on init to create database) + */ +#define KISSDB_ERROR_INVALID_PARAMETERS -3 + +/** + * Database file appears corrupt + */ +#define KISSDB_ERROR_CORRUPT_DBFILE -4 + +/** + * Open mode: read only + */ +#define KISSDB_OPEN_MODE_RDONLY 1 + +/** + * Open mode: read/write + */ +#define KISSDB_OPEN_MODE_RDWR 2 + +/** + * Open mode: read/write, create if doesn't exist + */ +#define KISSDB_OPEN_MODE_RWCREAT 3 + +/** + * Open mode: truncate database, open for reading and writing + */ +#define KISSDB_OPEN_MODE_RWREPLACE 4 + +/** + * Open database + * + * The three _size parameters must be specified if the database could + * be created or re-created. Otherwise an error will occur. If the + * database already exists, these parameters are ignored and are read + * from the database. You can check the struture afterwords to see what + * they were. + * + * @param db Database struct + * @param path Path to file + * @param mode One of the KISSDB_OPEN_MODE constants + * @param hash_table_size Size of hash table in 64-bit entries (must be >0) + * @param key_size Size of keys in bytes + * @param value_size Size of values in bytes + * @return 0 on success, nonzero on error + */ +extern int KISSDB_open( + KISSDB *db, + const char *path, + int mode, + unsigned long hash_table_size, + unsigned long key_size, + unsigned long value_size); + +/** + * Close database + * + * @param db Database struct + */ +extern void KISSDB_close(KISSDB *db); + +/** + * Get an entry + * + * @param db Database struct + * @param key Key (key_size bytes) + * @param vbuf Value buffer (value_size bytes capacity) + * @return -1 on I/O error, 0 on success, 1 on not found + */ +extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); + +/** + * Put an entry (overwriting it if it already exists) + * + * In the already-exists case the size of the database file does not + * change. + * + * @param db Database struct + * @param key Key (key_size bytes) + * @param value Value (value_size bytes) + * @return -1 on I/O error, 0 on success + */ +extern int KISSDB_put(KISSDB *db,const void *key,const void *value); + +/** + * Cursor used for iterating over all entries in database + */ +typedef struct { + KISSDB *db; + unsigned long h_no; + unsigned long h_idx; +} KISSDB_Iterator; + +/** + * Initialize an iterator + * + * @param db Database struct + * @param i Iterator to initialize + */ +extern void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi); + +/** + * Get the next entry + * + * The order of entries returned by iterator is undefined. It depends on + * how keys hash. + * + * @param Database iterator + * @param kbuf Buffer to fill with next key (key_size bytes) + * @param vbuf Buffer to fill with next value (value_size bytes) + * @return 0 if there are no more entries, negative on error, positive if an kbuf/vbuf have been filled + */ +extern int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/ext/vsdm/LICENSE.txt b/ext/vsdm/LICENSE.txt new file mode 100644 index 00000000..00b65473 --- /dev/null +++ b/ext/vsdm/LICENSE.txt @@ -0,0 +1,22 @@ +MIT LICENSE + +Copyright 2017 ZeroTier, Inc. +https://www.zerotier.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/ext/vsdm/Makefile b/ext/vsdm/Makefile new file mode 100644 index 00000000..828d6edc --- /dev/null +++ b/ext/vsdm/Makefile @@ -0,0 +1,5 @@ +all: + c++ -Os -std=c++11 -o vsdm-test vsdm-test.cpp + +clean: + rm -f vsdm-test *.o *.dSYM diff --git a/ext/vsdm/README.md b/ext/vsdm/README.md new file mode 100644 index 00000000..03354be6 --- /dev/null +++ b/ext/vsdm/README.md @@ -0,0 +1,40 @@ +VSDM: Very Simple Distributed Map +====== + +VSDM is a super-minimal replicated in-memory associative container. Its advantages are small code size, small footprint, simplicity, and lack of dependencies. + +VSDM uses a rumor mill replication algorithm that provides fast best-effort replication. If connectivity is stable this results in eventual consistency, but data loss or regression can occur under split brain conditions. This class is not recommended for data that is intolerant of loss or regression. Its ideal use case is a distributed cache for small data objects or a distributed database for ephemeral data. + +Transport is via TCP and can optionally be encrypted if one specifies a cryptor class (see below). The transport protocol does not implement any features for versioning or backward compatibility and changes to key/value type, cryptor, or other relevant parameters will render it incompatible. Again this is designed for simple app-embedded use cases. Use something more feature complete if you need to support different versions across the network. + +Each node maintains a 64-bit monotonically increasing revision counter that starts at zero. When a node connects to another node it sends this revision counter and the other party will send all updates with revision numbers greater than or equal to it. When a node receives an update it replaces the entry it has if the revision counter is higher and also sets its own revision counter to the new counter if it is higher. It then re-transmits the update if a replace event occurred (if the new update was newer). This is the "rumor mill" part: each node retransmits all changes to all other connected nodes (excluding the source). + +VSDM nodes can be connected according to any arbitrary connectivity graph. A more fully connected graph trades increased bandwidth consumption (due to redundant messages) for decreased likelihood of data loss or split brain conditions. + +The VSDM class is thread safe. It launches a single background thread to handle network I/O and periodic cleanup. Deleted keys are purged from memory after a period of time to allow time for propagation and possible re-propagation. + +## Template parameters + +VSDM supports a number of template parameters for customization. The only required parameters are K and V, the key and value types. The default serializer allows these to be one of the *uintX_t* stdint numeric types or *std::string*. Specify a different serializer for anything else. + +Here are the other parameters after K and V (in order): + + * **L**: Maximum message length, which also limits the max size of the combined key and value for a given entry. This imposes a sanity limit to prevent memory exhaustion. Default is 131072. Absolute max is UINT32_MAX - 4 (4294967291). + * **W**: Watcher function type. Default is `vsdm_watcher_noop` which does nothing. The watcher function (or function object) is passed into the constructor and receives notifications of remotely initiated changes to the map's contents. (Local changes via set() or del() do not trigger the watcher). The watcher must have the following methods: + * `void add(uint64_t remoteNodeId,const K &key,const V &value,uint64_t revision)` + * `void update(uint64_t remoteNodeId,const K &key,const V &value,uint64_t revision)` + * `void del(uint64_t remoteNodeId,const K &key)` + * **S**: Serializer class, which must contain *static* methods for serializing and deserializing keys and values. The following must be present for both key and value types: + * `unsigned long objectSize(const [K|V] &)` + * `const char *objectData(const [K|V] &)` + * `bool objectDeserialize(const char *,unsigned long,[K|V] &)` (false return means object was invalid and causes disconnect) + * **C**: Cryptor type to encrypt transport, default is `vsdm_cryptor_noop` (no encryption). It is also passed into the constructor for internal initialization. This must implement a static method `static unsigned long overhead()` that returns the *constant* per-message overhead for things like IV and MAC, and two methods to encrypt and decrypt the payload in place. The vsdm class will add space for overhead at the *end* of the message. Encrypt/decrypt mathods have these signatures: + * `void encrypt(void *,unsigned long)` + * `bool decrypt(void *,unsigned long)` (false return means invalid MAC and causes disconnect) + * **M**: Map type for underlying storage. Default is `std::unordered_map` with default STL hashers. Substitute a different container if you need to deal with non-hashable keys or need a sorted map. Containers supporting duplicate keys should not be used as the replication algorithm will not function properly. + +## License + +(c)2017 ZeroTier, Inc. (MIT license) + +Written by [Adam Ierymenko](https://github.com/adamierymenko) diff --git a/ext/vsdm/vsdm-test.cpp b/ext/vsdm/vsdm-test.cpp new file mode 100644 index 00000000..13c49afe --- /dev/null +++ b/ext/vsdm/vsdm-test.cpp @@ -0,0 +1,72 @@ +#include +#include +#include + +#include + +#define VSDM_DEBUG 1 + +#include "vsdm.hpp" + +int main(int argc,char **argv) +{ + if (argc < 4) { + printf("Usage: vsdm-test [//]\n"); + return 0; + } + + uint64_t id = (uint64_t)strtoull(argv[1],(char **)0,10); + uint64_t node = (uint64_t)strtoull(argv[2],(char **)0,10); + int port = (int)strtol(argv[3],(char **)0,10); + + struct sockaddr_in sa; + memset(&sa,0,sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_port = htons((uint16_t)port); + + vsdm m(id,node,false); + m.listen((const struct sockaddr *)&sa); + + for(int i=4;i + * License: MIT + */ + +#ifndef ZT_VSDM_HPP__ +#define ZT_VSDM_HPP__ + +#include +#include +#include + +#if defined(_WIN32) || defined(_WIN64) + +#include +#include +#include + +#define ZT_PHY_SOCKFD_TYPE SOCKET +#define ZT_PHY_SOCKFD_NULL (INVALID_SOCKET) +#define ZT_PHY_SOCKFD_VALID(s) ((s) != INVALID_SOCKET) +#define ZT_PHY_CLOSE_SOCKET(s) ::closesocket(s) +#define ZT_PHY_MAX_SOCKETS (FD_SETSIZE) +#define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS +#define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage + +#else // not Windows + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ZT_PHY_SOCKFD_TYPE int +#define ZT_PHY_SOCKFD_NULL (-1) +#define ZT_PHY_SOCKFD_VALID(s) ((s) > -1) +#define ZT_PHY_CLOSE_SOCKET(s) ::close(s) +#define ZT_PHY_MAX_SOCKETS (FD_SETSIZE) +#define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS +#define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage + +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/*********************************************************************************************************/ + +namespace ztVsdmInternal { + +/* This is the Phy<> adapter implementation for selected sockets from ZeroTier One. + * It should build and run out of the box on Windows and most *nix systems. Parts + * not used by VSDM have been removed. */ + +typedef void PhySocket; + +template +class Phy +{ +private: + HANDLER_PTR_TYPE _handler; + + enum PhySocketType + { + ZT_PHY_SOCKET_CLOSED = 0x00, // socket is closed, will be removed on next poll() + ZT_PHY_SOCKET_TCP_OUT_PENDING = 0x01, + ZT_PHY_SOCKET_TCP_OUT_CONNECTED = 0x02, + ZT_PHY_SOCKET_TCP_IN = 0x03, + ZT_PHY_SOCKET_TCP_LISTEN = 0x04, + ZT_PHY_SOCKET_UDP = 0x05, + ZT_PHY_SOCKET_FD = 0x06, + ZT_PHY_SOCKET_UNIX_IN = 0x07, + ZT_PHY_SOCKET_UNIX_LISTEN = 0x08 + }; + + struct PhySocketImpl + { + PhySocketType type; + ZT_PHY_SOCKFD_TYPE sock; + void *uptr; // user-settable pointer + ZT_PHY_SOCKADDR_STORAGE_TYPE saddr; // remote for TCP_OUT and TCP_IN, local for TCP_LISTEN, RAW, and UDP + }; + + std::list _socks; + fd_set _readfds; + fd_set _writefds; +#if defined(_WIN32) || defined(_WIN64) + fd_set _exceptfds; +#endif + long _nfds; + + ZT_PHY_SOCKFD_TYPE _whackReceiveSocket; + ZT_PHY_SOCKFD_TYPE _whackSendSocket; + + bool _noDelay; + bool _noCheck; + +public: + /** + * @param handler Pointer of type HANDLER_PTR_TYPE to handler + * @param noDelay If true, disable TCP NAGLE algorithm on TCP sockets + * @param noCheck If true, attempt to set UDP SO_NO_CHECK option to disable sending checksums + */ + Phy(HANDLER_PTR_TYPE handler,bool noDelay,bool noCheck) : + _handler(handler) + { + FD_ZERO(&_readfds); + FD_ZERO(&_writefds); + +#if defined(_WIN32) || defined(_WIN64) + FD_ZERO(&_exceptfds); + + SOCKET pipes[2]; + { // hack copied from StackOverflow, behaves a bit like pipe() on *nix systems + struct sockaddr_in inaddr; + struct sockaddr addr; + SOCKET lst=::socket(AF_INET, SOCK_STREAM,IPPROTO_TCP); + if (lst == INVALID_SOCKET) + throw std::runtime_error("unable to create pipes for select() abort"); + memset(&inaddr, 0, sizeof(inaddr)); + memset(&addr, 0, sizeof(addr)); + inaddr.sin_family = AF_INET; + inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + inaddr.sin_port = 0; + int yes=1; + setsockopt(lst,SOL_SOCKET,SO_REUSEADDR,(char*)&yes,sizeof(yes)); + bind(lst,(struct sockaddr *)&inaddr,sizeof(inaddr)); + listen(lst,1); + int len=sizeof(inaddr); + getsockname(lst, &addr,&len); + pipes[0]=::socket(AF_INET, SOCK_STREAM,0); + if (pipes[0] == INVALID_SOCKET) + throw std::runtime_error("unable to create pipes for select() abort"); + connect(pipes[0],&addr,len); + pipes[1]=accept(lst,0,0); + closesocket(lst); + } +#else // not Windows + int pipes[2]; + if (::pipe(pipes)) + throw std::runtime_error("unable to create pipes for select() abort"); +#endif // Windows or not + + _nfds = (pipes[0] > pipes[1]) ? (long)pipes[0] : (long)pipes[1]; + _whackReceiveSocket = pipes[0]; + _whackSendSocket = pipes[1]; + _noDelay = noDelay; + _noCheck = noCheck; + } + + ~Phy() + { + for(typename std::list::const_iterator s(_socks.begin());s!=_socks.end();++s) { + if (s->type != ZT_PHY_SOCKET_CLOSED) + this->close((PhySocket *)&(*s),true); + } + ZT_PHY_CLOSE_SOCKET(_whackReceiveSocket); + ZT_PHY_CLOSE_SOCKET(_whackSendSocket); + } + + /** + * @param s Socket object + * @return Underlying OS-type (usually int or long) file descriptor associated with object + */ + static inline ZT_PHY_SOCKFD_TYPE getDescriptor(PhySocket *s) throw() { return reinterpret_cast(s)->sock; } + + /** + * @param s Socket object + * @return Pointer to user object + */ + static inline void** getuptr(PhySocket *s) throw() { return &(reinterpret_cast(s)->uptr); } + + /** + * Cause poll() to stop waiting immediately + * + * This can be used to reset the polling loop after changes that require + * attention, or to shut down a background thread that is waiting, etc. + */ + inline void whack() + { +#if defined(_WIN32) || defined(_WIN64) + ::send(_whackSendSocket,(const char *)this,1,0); +#else + (void)(::write(_whackSendSocket,(PhySocket *)this,1)); +#endif + } + + /** + * @return Number of open sockets + */ + inline unsigned long count() const throw() { return _socks.size(); } + + /** + * @return Maximum number of sockets allowed + */ + inline unsigned long maxCount() const throw() { return ZT_PHY_MAX_SOCKETS; } + + /** + * Bind a local listen socket to listen for new TCP connections + * + * @param localAddress Local address and port + * @param uptr Initial value of uptr for new socket (default: NULL) + * @return Socket or NULL on failure to bind + */ + inline PhySocket *tcpListen(const struct sockaddr *localAddress,void *uptr = (void *)0) + { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) + return (PhySocket *)0; + + ZT_PHY_SOCKFD_TYPE s = ::socket(localAddress->sa_family,SOCK_STREAM,0); + if (!ZT_PHY_SOCKFD_VALID(s)) + return (PhySocket *)0; + +#if defined(_WIN32) || defined(_WIN64) + { + BOOL f; + f = TRUE; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(const char *)&f,sizeof(f)); + f = TRUE; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(const char *)&f,sizeof(f)); + f = (_noDelay ? TRUE : FALSE); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + u_long iMode=1; + ioctlsocket(s,FIONBIO,&iMode); + } +#else + { + int f; + f = 1; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f)); + f = 1; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f)); + f = (_noDelay ? 1 : 0); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + fcntl(s,F_SETFL,O_NONBLOCK); + } +#endif + + if (::bind(s,localAddress,(localAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in))) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + + if (::listen(s,1024)) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + + try { + _socks.push_back(PhySocketImpl()); + } catch ( ... ) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + PhySocketImpl &sws = _socks.back(); + + if ((long)s > _nfds) + _nfds = (long)s; + FD_SET(s,&_readfds); + sws.type = ZT_PHY_SOCKET_TCP_LISTEN; + sws.sock = s; + sws.uptr = uptr; + memset(&(sws.saddr),0,sizeof(struct sockaddr_storage)); + memcpy(&(sws.saddr),localAddress,(localAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); + + return (PhySocket *)&sws; + } + + /** + * Start a non-blocking connect; CONNECT handler is called on success or failure + * + * A return value of NULL indicates a synchronous failure such as a + * failure to open a socket. The TCP connection handler is not called + * in this case. + * + * It is possible on some platforms for an "instant connect" to occur, + * such as when connecting to a loopback address. In this case, the + * 'connected' result parameter will be set to 'true' and if the + * 'callConnectHandler' flag is true (the default) the TCP connect + * handler will be called before the function returns. + * + * These semantics can be a bit confusing, but they're less so than + * the underlying semantics of asynchronous TCP connect. + * + * @param remoteAddress Remote address + * @param connected Result parameter: set to whether an "instant connect" has occurred (true if yes) + * @param uptr Initial value of uptr for new socket (default: NULL) + * @param callConnectHandler If true, call TCP connect handler even if result is known before function exit (default: true) + * @return New socket or NULL on failure + */ + inline PhySocket *tcpConnect(const struct sockaddr *remoteAddress,bool &connected,void *uptr = (void *)0,bool callConnectHandler = true) + { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) + return (PhySocket *)0; + + ZT_PHY_SOCKFD_TYPE s = ::socket(remoteAddress->sa_family,SOCK_STREAM,0); + if (!ZT_PHY_SOCKFD_VALID(s)) { + connected = false; + return (PhySocket *)0; + } + +#if defined(_WIN32) || defined(_WIN64) + { + BOOL f; + if (remoteAddress->sa_family == AF_INET6) { f = TRUE; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(const char *)&f,sizeof(f)); } + f = TRUE; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(const char *)&f,sizeof(f)); + f = (_noDelay ? TRUE : FALSE); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + u_long iMode=1; + ioctlsocket(s,FIONBIO,&iMode); + } +#else + { + int f; + if (remoteAddress->sa_family == AF_INET6) { f = 1; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f)); } + f = 1; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f)); + f = (_noDelay ? 1 : 0); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + fcntl(s,F_SETFL,O_NONBLOCK); + } +#endif + + connected = true; + if (::connect(s,remoteAddress,(remoteAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in))) { + connected = false; +#if defined(_WIN32) || defined(_WIN64) + if (WSAGetLastError() != WSAEWOULDBLOCK) { +#else + if (errno != EINPROGRESS) { +#endif + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } // else connection is proceeding asynchronously... + } + + try { + _socks.push_back(PhySocketImpl()); + } catch ( ... ) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + PhySocketImpl &sws = _socks.back(); + + if ((long)s > _nfds) + _nfds = (long)s; + if (connected) { + FD_SET(s,&_readfds); + sws.type = ZT_PHY_SOCKET_TCP_OUT_CONNECTED; + } else { + FD_SET(s,&_writefds); +#if defined(_WIN32) || defined(_WIN64) + FD_SET(s,&_exceptfds); +#endif + sws.type = ZT_PHY_SOCKET_TCP_OUT_PENDING; + } + sws.sock = s; + sws.uptr = uptr; + memset(&(sws.saddr),0,sizeof(struct sockaddr_storage)); + memcpy(&(sws.saddr),remoteAddress,(remoteAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); + + if ((callConnectHandler)&&(connected)) { + try { + _handler->phyOnTcpConnect((PhySocket *)&sws,&(sws.uptr),true); + } catch ( ... ) {} + } + + return (PhySocket *)&sws; + } + + /** + * Attempt to send data to a stream socket (non-blocking) + * + * If -1 is returned, the socket should no longer be used as it is now + * destroyed. If callCloseHandler is true, the close handler will be + * called before the function returns. + * + * This can be used with TCP, Unix, or socket pair sockets. + * + * @param sock An open stream socket (other socket types will fail) + * @param data Data to send + * @param len Length of data + * @param callCloseHandler If true, call close handler on socket closing failure condition (default: true) + * @return Number of bytes actually sent or -1 on fatal error (socket closure) + */ + inline long streamSend(PhySocket *sock,const void *data,unsigned long len,bool callCloseHandler = true) + { + PhySocketImpl &sws = *(reinterpret_cast(sock)); +#if defined(_WIN32) || defined(_WIN64) + long n = (long)::send(sws.sock,reinterpret_cast(data),len,0); + if (n == SOCKET_ERROR) { + switch(WSAGetLastError()) { + case WSAEINTR: + case WSAEWOULDBLOCK: + return 0; + default: + this->close(sock,callCloseHandler); + return -1; + } + } +#else // not Windows + long n = (long)::send(sws.sock,data,len,0); + if (n < 0) { + switch(errno) { +#ifdef EAGAIN + case EAGAIN: +#endif +#if defined(EWOULDBLOCK) && ( !defined(EAGAIN) || (EWOULDBLOCK != EAGAIN) ) + case EWOULDBLOCK: +#endif +#ifdef EINTR + case EINTR: +#endif + return 0; + default: + this->close(sock,callCloseHandler); + return -1; + } + } +#endif // Windows or not + return n; + } + + /** + * For streams, sets whether we want to be notified that the socket is writable + * + * This can be used with TCP, Unix, or socket pair sockets. + * + * Call whack() if this is being done from another thread and you want + * it to take effect immediately. Otherwise it is only guaranteed to + * take effect on the next poll(). + * + * @param sock Stream connection socket + * @param notifyWritable Want writable notifications? + */ + inline const void setNotifyWritable(PhySocket *sock,bool notifyWritable) + { + PhySocketImpl &sws = *(reinterpret_cast(sock)); + if (notifyWritable) { + FD_SET(sws.sock,&_writefds); + } else { + FD_CLR(sws.sock,&_writefds); + } + } + + /** + * Set whether we want to be notified that a socket is readable + * + * This is primarily for raw sockets added with wrapSocket(). It could be + * used with others, but doing so would essentially lock them and prevent + * data from being read from them until this is set to 'true' again. + * + * @param sock Socket to modify + * @param notifyReadable True if socket should be monitored for readability + */ + inline const void setNotifyReadable(PhySocket *sock,bool notifyReadable) + { + PhySocketImpl &sws = *(reinterpret_cast(sock)); + if (notifyReadable) { + FD_SET(sws.sock,&_readfds); + } else { + FD_CLR(sws.sock,&_readfds); + } + } + + /** + * Wait for activity and handle one or more events + * + * Note that this is not guaranteed to wait up to 'timeout' even + * if nothing happens, as whack() or other events such as signals + * may cause premature termination. + * + * @param timeout Timeout in milliseconds or 0 for none (forever) + */ + inline void poll(unsigned long timeout) + { + char buf[131072]; + struct sockaddr_storage ss; + struct timeval tv; + fd_set rfds,wfds,efds; + + memcpy(&rfds,&_readfds,sizeof(rfds)); + memcpy(&wfds,&_writefds,sizeof(wfds)); +#if defined(_WIN32) || defined(_WIN64) + memcpy(&efds,&_exceptfds,sizeof(efds)); +#else + FD_ZERO(&efds); +#endif + + tv.tv_sec = (long)(timeout / 1000); + tv.tv_usec = (long)((timeout % 1000) * 1000); + if (::select((int)_nfds + 1,&rfds,&wfds,&efds,(timeout > 0) ? &tv : (struct timeval *)0) <= 0) + return; + + if (FD_ISSET(_whackReceiveSocket,&rfds)) { + char tmp[16]; +#if defined(_WIN32) || defined(_WIN64) + ::recv(_whackReceiveSocket,tmp,16,0); +#else + ::read(_whackReceiveSocket,tmp,16); +#endif + } + + for(typename std::list::iterator s(_socks.begin());s!=_socks.end();) { + switch (s->type) { + + case ZT_PHY_SOCKET_TCP_OUT_PENDING: +#if defined(_WIN32) || defined(_WIN64) + if (FD_ISSET(s->sock,&efds)) { + this->close((PhySocket *)&(*s),true); + } else // ... if +#endif + if (FD_ISSET(s->sock,&wfds)) { + socklen_t slen = sizeof(ss); + if (::getpeername(s->sock,(struct sockaddr *)&ss,&slen) != 0) { + this->close((PhySocket *)&(*s),true); + } else { + s->type = ZT_PHY_SOCKET_TCP_OUT_CONNECTED; + FD_SET(s->sock,&_readfds); + FD_CLR(s->sock,&_writefds); +#if defined(_WIN32) || defined(_WIN64) + FD_CLR(s->sock,&_exceptfds); +#endif + try { + _handler->phyOnTcpConnect((PhySocket *)&(*s),&(s->uptr),true); + } catch ( ... ) {} + } + } + break; + + case ZT_PHY_SOCKET_TCP_OUT_CONNECTED: + case ZT_PHY_SOCKET_TCP_IN: { + ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable + if (FD_ISSET(sock,&rfds)) { + long n = (long)::recv(sock,buf,sizeof(buf),0); + if (n <= 0) { + this->close((PhySocket *)&(*s),true); + } else { + try { + _handler->phyOnTcpData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n); + } catch ( ... ) {} + } + } + if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { + try { + _handler->phyOnTcpWritable((PhySocket *)&(*s),&(s->uptr)); + } catch ( ... ) {} + } + } break; + + case ZT_PHY_SOCKET_TCP_LISTEN: + if (FD_ISSET(s->sock,&rfds)) { + memset(&ss,0,sizeof(ss)); + socklen_t slen = sizeof(ss); + ZT_PHY_SOCKFD_TYPE newSock = ::accept(s->sock,(struct sockaddr *)&ss,&slen); + if (ZT_PHY_SOCKFD_VALID(newSock)) { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) { + ZT_PHY_CLOSE_SOCKET(newSock); + } else { +#if defined(_WIN32) || defined(_WIN64) + { BOOL f = (_noDelay ? TRUE : FALSE); setsockopt(newSock,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); } + { u_long iMode=1; ioctlsocket(newSock,FIONBIO,&iMode); } +#else + { int f = (_noDelay ? 1 : 0); setsockopt(newSock,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); } + fcntl(newSock,F_SETFL,O_NONBLOCK); +#endif + _socks.push_back(PhySocketImpl()); + PhySocketImpl &sws = _socks.back(); + FD_SET(newSock,&_readfds); + if ((long)newSock > _nfds) + _nfds = (long)newSock; + sws.type = ZT_PHY_SOCKET_TCP_IN; + sws.sock = newSock; + sws.uptr = (void *)0; + memcpy(&(sws.saddr),&ss,sizeof(struct sockaddr_storage)); + try { + _handler->phyOnTcpAccept((PhySocket *)&(*s),(PhySocket *)&(_socks.back()),&(s->uptr),&(sws.uptr),(const struct sockaddr *)&(sws.saddr)); + } catch ( ... ) {} + } + } + } + break; + + case ZT_PHY_SOCKET_UDP: + if (FD_ISSET(s->sock,&rfds)) { + for(;;) { + memset(&ss,0,sizeof(ss)); + socklen_t slen = sizeof(ss); + long n = (long)::recvfrom(s->sock,buf,sizeof(buf),0,(struct sockaddr *)&ss,&slen); + if (n > 0) { + try { + _handler->phyOnDatagram((PhySocket *)&(*s),&(s->uptr),(const struct sockaddr *)&(s->saddr),(const struct sockaddr *)&ss,(void *)buf,(unsigned long)n); + } catch ( ... ) {} + } else if (n < 0) + break; + } + } + break; + + case ZT_PHY_SOCKET_UNIX_IN: { +#ifdef __UNIX_LIKE__ + ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable + if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { + try { + _handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr),false); + } catch ( ... ) {} + } + if (FD_ISSET(sock,&rfds)) { + long n = (long)::read(sock,buf,sizeof(buf)); + if (n <= 0) { + this->close((PhySocket *)&(*s),true); + } else { + try { + _handler->phyOnUnixData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n); + } catch ( ... ) {} + } + } +#endif // __UNIX_LIKE__ + } break; + + case ZT_PHY_SOCKET_UNIX_LISTEN: +#ifdef __UNIX_LIKE__ + if (FD_ISSET(s->sock,&rfds)) { + memset(&ss,0,sizeof(ss)); + socklen_t slen = sizeof(ss); + ZT_PHY_SOCKFD_TYPE newSock = ::accept(s->sock,(struct sockaddr *)&ss,&slen); + if (ZT_PHY_SOCKFD_VALID(newSock)) { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) { + ZT_PHY_CLOSE_SOCKET(newSock); + } else { + fcntl(newSock,F_SETFL,O_NONBLOCK); + _socks.push_back(PhySocketImpl()); + PhySocketImpl &sws = _socks.back(); + FD_SET(newSock,&_readfds); + if ((long)newSock > _nfds) + _nfds = (long)newSock; + sws.type = ZT_PHY_SOCKET_UNIX_IN; + sws.sock = newSock; + sws.uptr = (void *)0; + memcpy(&(sws.saddr),&ss,sizeof(struct sockaddr_storage)); + try { + //_handler->phyOnUnixAccept((PhySocket *)&(*s),(PhySocket *)&(_socks.back()),&(s->uptr),&(sws.uptr)); + } catch ( ... ) {} + } + } + } +#endif // __UNIX_LIKE__ + break; + + case ZT_PHY_SOCKET_FD: { + ZT_PHY_SOCKFD_TYPE sock = s->sock; + const bool readable = ((FD_ISSET(sock,&rfds))&&(FD_ISSET(sock,&_readfds))); + const bool writable = ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))); + if ((readable)||(writable)) { + try { + //_handler->phyOnFileDescriptorActivity((PhySocket *)&(*s),&(s->uptr),readable,writable); + } catch ( ... ) {} + } + } break; + + default: + break; + + } + + if (s->type == ZT_PHY_SOCKET_CLOSED) + _socks.erase(s++); + else ++s; + } + } + + /** + * @param sock Socket to close + * @param callHandlers If true, call handlers for TCP connect (success: false) or close (default: true) + */ + inline void close(PhySocket *sock,bool callHandlers = true) + { + if (!sock) + return; + PhySocketImpl &sws = *(reinterpret_cast(sock)); + if (sws.type == ZT_PHY_SOCKET_CLOSED) + return; + + FD_CLR(sws.sock,&_readfds); + FD_CLR(sws.sock,&_writefds); +#if defined(_WIN32) || defined(_WIN64) + FD_CLR(sws.sock,&_exceptfds); +#endif + + if (sws.type != ZT_PHY_SOCKET_FD) + ZT_PHY_CLOSE_SOCKET(sws.sock); + +#ifdef __UNIX_LIKE__ + if (sws.type == ZT_PHY_SOCKET_UNIX_LISTEN) + ::unlink(((struct sockaddr_un *)(&(sws.saddr)))->sun_path); +#endif // __UNIX_LIKE__ + + if (callHandlers) { + switch(sws.type) { + case ZT_PHY_SOCKET_TCP_OUT_PENDING: + try { + _handler->phyOnTcpConnect(sock,&(sws.uptr),false); + } catch ( ... ) {} + break; + case ZT_PHY_SOCKET_TCP_OUT_CONNECTED: + case ZT_PHY_SOCKET_TCP_IN: + try { + _handler->phyOnTcpClose(sock,&(sws.uptr)); + } catch ( ... ) {} + break; + case ZT_PHY_SOCKET_UNIX_IN: +#ifdef __UNIX_LIKE__ + try { + _handler->phyOnUnixClose(sock,&(sws.uptr)); + } catch ( ... ) {} +#endif // __UNIX_LIKE__ + break; + default: + break; + } + } + + // Causes entry to be deleted from list in poll(), ignored elsewhere + sws.type = ZT_PHY_SOCKET_CLOSED; + + if ((long)sws.sock >= (long)_nfds) { + long nfds = (long)_whackSendSocket; + if ((long)_whackReceiveSocket > nfds) + nfds = (long)_whackReceiveSocket; + for(typename std::list::iterator s(_socks.begin());s!=_socks.end();++s) { + if ((s->type != ZT_PHY_SOCKET_CLOSED)&&((long)s->sock > nfds)) + nfds = (long)s->sock; + } + _nfds = nfds; + } + } +}; + +static inline uint64_t _swap64(const uint64_t n) +{ + return ( + ((n & 0x00000000000000FFULL) << 56) | + ((n & 0x000000000000FF00ULL) << 40) | + ((n & 0x0000000000FF0000ULL) << 24) | + ((n & 0x00000000FF000000ULL) << 8) | + ((n & 0x000000FF00000000ULL) >> 8) | + ((n & 0x0000FF0000000000ULL) >> 24) | + ((n & 0x00FF000000000000ULL) >> 40) | + ((n & 0xFF00000000000000ULL) >> 56) + ); +} + +} // namespace ztVsdmInternal + +/*********************************************************************************************************/ + +/** + * No-op update watcher + */ +class vsdm_watcher_noop +{ +public: + template + inline void add(uint64_t,const K &k,const V &v,uint64_t) {} + template + inline void update(uint64_t,const K &k,const V &v,uint64_t) {} + template + inline void del(uint64_t,const K &k) {} +}; + +/** + * No-op cryptor that adds no overhead and does no encryption + */ +class vsdm_cryptor_noop +{ +public: + static inline unsigned long overhead() { return 0; } + inline void encrypt(void *d,unsigned long l) {} + inline bool decrypt(void *d,unsigned long l) { return true; } +}; + +/** + * Default serializer supporting std::string and stdint.h types + */ +class vsdm_default_serializer +{ +public: + static inline unsigned long objectSize(const std::string &o) { return o.length(); } + static inline unsigned long objectSize(const uint8_t o) { return 1; } + static inline unsigned long objectSize(const int8_t o) { return 1; } + static inline unsigned long objectSize(const uint16_t o) { return 2; } + static inline unsigned long objectSize(const int16_t o) { return 2; } + static inline unsigned long objectSize(const uint32_t o) { return 4; } + static inline unsigned long objectSize(const int32_t o) { return 4; } + static inline unsigned long objectSize(const uint64_t o) { return 8; } + static inline unsigned long objectSize(const int64_t o) { return 8; } + + static inline const char *objectData(const std::string &o) { return o.data(); } + static inline const char *objectData(const uint8_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int8_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const uint16_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int16_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const uint32_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int32_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const uint64_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int64_t &o) { return reinterpret_cast(&o); } + + static inline bool objectDeserialize(const char *d,unsigned long l,std::string &o) { o.assign(d,l); return true; } + static inline bool objectDeserialize(const char *d,unsigned long l,uint8_t &o) { if (l == 1) { memcpy(&o,d,1); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int8_t &o) { if (l == 1) { memcpy(&o,d,1); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,uint16_t &o) { if (l == 2) { memcpy(&o,d,2); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int16_t &o) { if (l == 2) { memcpy(&o,d,2); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,uint32_t &o) { if (l == 4) { memcpy(&o,d,4); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int32_t &o) { if (l == 4) { memcpy(&o,d,4); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,uint64_t &o) { if (l == 8) { memcpy(&o,d,8); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int64_t &o) { if (l == 8) { memcpy(&o,d,8); return true; } else { return false; } } +}; + +/** + * VSDM: Very Simple Distributed Map + * + * See README.md for full docs. + * + * @tparam K Key type (must be supported by serializer) + * @tparam V Value type (must be supported by serializer) + * @tparam L Maximum message length (max allowed: UINT32_MAX - 1, default: 131072) + * @tparam W Watcher function (default: vsdm_watcher_noop) + * @tparam S Serializer class with static methods to serialize keys and values (default: vsdm_default_serializer) + * @tparam C Cryptor to encrypt/decrypt and authenticate network traffic (default: vsdm_cryptor_noop) + * @tparam M Map type for underlying data store (default: std::unordered_map) + */ +template< + typename K, + typename V, + unsigned long L = 131072, + typename W = vsdm_watcher_noop, + typename S = vsdm_default_serializer, + typename C = vsdm_cryptor_noop, + template class M = std::unordered_map +> +class vsdm +{ + friend void vsdm_thread_main(void *parent); + friend class ztVsdmInternal::Phy; + +private: + struct vsdm_entry + { + vsdm_entry() : rev(0),deletedAt(0),v() {} + uint64_t rev; + uint64_t deletedAt; + V v; + }; + + struct _connection + { + _connection() : outbuf(),inbuf(),gotHello(false),node(0),sock((ztVsdmInternal::PhySocket *)0) {} + std::string outbuf; + std::string inbuf; + bool gotHello; + uint64_t node; + ztVsdmInternal::PhySocket *sock; + }; + +public: + typedef K key_type; + typedef V value_type; + + /** + * @param id Cluster ID, must be the same on all nodes + * @param node Arbitrary unique node ID + * @param restrictInbound If true, restrict inbound connections to known peer IPs (added via link()) + * @param cryptor Encryptor/decryptor instance (default: C()) + * @param watcher Watcher function instance (default: W()) + */ + vsdm(uint64_t id,uint64_t node,bool restrictInbound,const C &cryptor = C(),const W &watcher = W()) : + _node(node), + _id(id), + _rev(0), + _connections(), + _m(), + _lock(), + _phy(this,false,false), + _cryptor(cryptor), + _watcher(watcher), + _run(true), + _restrictInbound(restrictInbound), + _t(_threadMain,reinterpret_cast(this)) + { + } + + ~vsdm() + { + _run = false; + _phy.whack(); + _t.join(); + } + + /** + * @param k Key to set + * @param v New value for key + * @return Revision of entry in map + */ + inline uint64_t set(const K &k,const V &v) + { + std::lock_guard l(_lock); + + vsdm_entry &e = _m[k]; + e.rev = ++_rev; + e.deletedAt = 0; + e.v = v; + + std::vector sentToNodes; + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,k,e); + sentToNodes.push_back(c2->second.node); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=%s\n",(unsigned long)c2->second.node,k.c_str(),v.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + + return _rev; + } + + /** + * @param k Key to check + * @return Revision of key that we have or 0 if not found + */ + inline uint64_t have(const K &k) const + { + std::lock_guard l(_lock); + typename std::unordered_map::const_iterator i(_m.find(k)); + if ((i == _m.end())||(i->second.deletedAt)) + return 0; + return i->second.rev; + } + + /** + * @param k Key to get + * @param dfl Default value if key is not found + * @param have If non-NULL, set to revision of this key or 0 if not found + * @return Key value or dfl if not found + */ + inline V get(const K &k,const V &dfl = V(),uint64_t *have = (uint64_t *)0) const + { + std::lock_guard l(_lock); + typename std::unordered_map::const_iterator i(_m.find(k)); + if ((i == _m.end())||(i->second.deletedAt)) { + if (have) + *have = 0; + return dfl; + } + if (have) + *have = i->second.rev; + return i->second.v; + } + + /** + * @param k Key to get + * @param have If non-NULL, set to revision of this key or 0 if not found + * @return Key's value or default/empty V() if not found + */ + inline V get(const K &k,uint64_t *have) const + { + return get(k,V(),have); + } + + /** + * Erase a key + * + * Erased entries are not wholly purged from memory immediately. They + * are marked as erased and purged after sufficient time for propagation. + * + * @param k Key to erase + * @return Previous revision of this key in map or 0 if not found + */ + inline bool del(const K &k) + { + uint64_t prev = 0; + std::lock_guard l(_lock); + + typename std::unordered_map::iterator i(_m.find(k)); + if (i == _m.end()) + return 0; + prev = i->second.rev; + i->second.rev = ++_rev; + i->second.deletedAt = _rev; + i->second.v.clear(); + + std::vector sentToNodes; + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,k,i->second); + sentToNodes.push_back(c2->second.node); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=\n",(unsigned long)c2->second.node,k.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + + return prev; + } + + /** + * Listen for incoming node connections on an address + * + * This can be called more than once to listen on more than one address and port. + * + * @param sa Socket address + * @return True if bind succeeded + */ + inline bool listen(const struct sockaddr *sa) + { + std::lock_guard l(_lock); + return (_phy.tcpListen(sa) != (ztVsdmInternal::PhySocket *)0); + } + + /** + * Add a remote node endpoint + * + * This can be called for an arbitrary number of other endpoints in the + * network to tell this node to attempt to maintain a link to them. + * + * @param node Node ID of remote + * @param sa Socket address of remote + * @param salen Length of socket address structure + */ + inline void link(uint64_t node,const struct sockaddr_in *sa,unsigned int salen) + { + std::lock_guard l(_lock); + if ((node != _node)&&(salen <= sizeof(struct sockaddr_storage))) + memcpy(&(_peers[node]),sa,salen); + } + + /** + * @return Node IDs of nodes that are currently connected + */ + inline std::vector who() const + { + std::vector w; + std::lock_guard l(_lock); + for(typename std::unordered_map::const_iterator i(_connections.begin());i!=_connections.end();++i) { + if ((i->gotHello)&&(std::find(w.begin(),w.end(),i->second.node) == w.end())) + w.push_back(i->second.node); + } + return w; + } + + /** + * @return True if we are currently connected to at least one other node + */ + inline bool connected() const + { + std::lock_guard l(_lock); + for(typename std::unordered_map::const_iterator i(_connections.begin());i!=_connections.end();++i) { + if (i->gotHello) + return true; + } + return false; + } + + /** + * Iterate through all members of this map, with optional deletion + * + * The function is executed against all key/value pairs and returns a signed integer. + * A negative return value causes the entry to be deleted, while a positive return + * value means the key's value (which is passed into the function as a reference) has + * been modified and should be replicated. A return value of zero means no change. + * + * Other methods should not be called since doing so can result in a deadlock. + * + * @param func Function to execute against all members of map, returns integer (see description) + */ + template + inline void each(F func) + { + std::vector sentToNodes; + bool whack = false; + std::lock_guard l(_lock); + for(typename M::iterator i(_m.begin());i!=_m.end();++i) { + if (!i->second.deletedAt) { + try { + const int result = func(i->first,i->second.v); + if (result < 0) { + i->second.rev = ++_rev; + i->second.deletedAt = _rev; + i->second.v.clear(); + } else if (result > 0) { + i->second.rev = ++_rev; + i->second.deletedAt = 0; + // v will have been modified in place + } + if (result != 0) { + sentToNodes.clear(); + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,i->first,i->second); + sentToNodes.push_back(c2->second.node); + } + } + whack = true; + } + } catch ( ... ) {} + } + } + if (whack) + _phy.whack(); + } + +private: + inline vsdm &operator=(const vsdm &v) { return *this; } + + static void _threadMain(void *p) { reinterpret_cast(p)->threadMain(); } + inline void threadMain() + { + std::vector haveNodes; + time_t lastcheck = 0; + time_t lastclean = 0; + while (_run) { + _phy.poll(1000); + + time_t now = time(0); + + // Check connections with other nodes and try to establish them + // if they're not present. + if ((now - lastcheck) >= 2) { + lastcheck = now; + haveNodes.clear(); + + std::lock_guard l(_lock); + + for(typename std::unordered_map::const_iterator c(_connections.begin());c!=_connections.end();++c) { + if (std::find(haveNodes.begin(),haveNodes.end(),c->second.node) == haveNodes.end()) + haveNodes.push_back(c->second.node); + } + + for(std::unordered_map::iterator p(_peers.begin());p!=_peers.end();++p) { + if (std::find(haveNodes.begin(),haveNodes.end(),p->first) == haveNodes.end()) { + bool connected = false; + ztVsdmInternal::PhySocket *ns = _phy.tcpConnect((const struct sockaddr *)&(p->second),connected,(void *)0,true); + if (ns) { + _connection &c = _connections[ns]; + c.gotHello = false; + c.node = p->first; + c.sock = ns; + } + } + } + } + + // Forget deleted entries if they've had ample time to propagate + if ((now - lastclean) >= 120) { + lastclean = now; + uint64_t delHorizon = _m.size() * (2 + _peers.size()); + if (_rev > delHorizon) { + delHorizon -= _rev; + std::lock_guard l(_lock); + for(typename M::iterator i(_m.begin());i!=_m.end();) { + if ((i->second.deletedAt > 0)&&(i->second.deletedAt < delHorizon)) + _m.erase(i++); + else ++i; + } + } + } + } + } + + inline void sendUpdate(_connection &c,const std::string &k,const vsdm_entry &e) + { + // assumes lock is locked + const uint32_t ks = (uint32_t)S::objectSize(k); + uint32_t vs = 0; + uint32_t hdr[4]; + hdr[0] = htonl((uint32_t)((e.rev >> 32) & 0xffffffff)); + hdr[1] = htonl((uint32_t)(e.rev & 0xffffffff)); + hdr[2] = htonl(ks); + if (e.deletedAt) { + hdr[3] = 0xffffffff; + } else { + vs = (uint32_t)S::objectSize(e.v); + hdr[3] = htonl(vs); + } + + const uint32_t s = htonl((uint32_t)(16 + C::overhead() + ks + vs)); + c.outbuf.append((const char *)&s,4); + + const unsigned long start = (unsigned long)c.outbuf.length(); + c.outbuf.append((const char *)hdr,16); + c.outbuf.append(S::objectData(k),ks); + if (!e.deletedAt) + c.outbuf.append(S::objectData(e.v),vs); + c.outbuf.append(C::overhead(),(char)0); + const unsigned long end = (unsigned long)c.outbuf.length(); + + _cryptor.encrypt(reinterpret_cast(const_cast(c.outbuf.data()) + start),end - start); + + _phy.setNotifyWritable(c.sock,true); + } + + inline void sendUpdateToAll(ztVsdmInternal::PhySocket *receivedOnSock,const uint64_t receivedFromNode,const std::string &k,const vsdm_entry &e) + { + // assumes lock is locked + std::vector sentToNodes; + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->first != receivedOnSock)&&(c2->second.gotHello)&&(c2->second.node != receivedFromNode)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,k,e); + sentToNodes.push_back(c2->second.node); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=%s\n",(unsigned long)c2->second.node,k.c_str(),(e.deletedAt) ? "" : e.v.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + } + + inline void sendHello(ztVsdmInternal::PhySocket *sock) + { + uint64_t hdr[3]; + if (htonl(1) == 1) { + hdr[0] = _node; + hdr[1] = _id; + hdr[2] = _rev; + } else { + hdr[0] = ztVsdmInternal::_swap64(_node); + hdr[1] = ztVsdmInternal::_swap64(_id); + hdr[2] = ztVsdmInternal::_swap64(_rev); + } + uint8_t tmp[24 + C::overhead()]; + memcpy(tmp,hdr,24); + _cryptor.encrypt(reinterpret_cast(tmp),sizeof(tmp)); + _phy.streamSend(sock,reinterpret_cast(tmp),sizeof(tmp)); + } + + inline void phyOnTcpConnect(ztVsdmInternal::PhySocket *sock,void **uptr,bool success) + { + std::lock_guard l(_lock); + if (success) { + _connection &c = _connections[sock]; + c.gotHello = false; + c.sock = sock; + *uptr = (void *)&c; + sendHello(sock); + } else { + _connections.erase(sock); + } + } + + inline void phyOnTcpAccept(ztVsdmInternal::PhySocket *sockL,ztVsdmInternal::PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from) + { + std::lock_guard l(_lock); + + if (_restrictInbound) { + bool ok = false; + for(typename std::unordered_map::const_iterator i(_peers.begin());i!=_peers.end();++i) { + if (from->sa_family == i->second.ss_family) { + if ( (from->sa_family == AF_INET) && (reinterpret_cast(from)->sin_addr.s_addr == reinterpret_cast(&(i->second))->sin_addr.s_addr) ) { + ok = true; + break; + } else if ( (from->sa_family == AF_INET6) && (memcmp(reinterpret_cast(from)->sin6_addr.s6_addr,reinterpret_cast(&(i->second))->sin6_addr.s6_addr,16) == 0) ) { + ok = true; + break; + } + } + } + if (!ok) { +#ifdef VSDM_DEBUG + fprintf(stderr," * dropped inbound connection: peer not from a known IP address\n"); fflush(stderr); +#endif + _phy.close(sockN,false); + return; + } + } + + _connection &c = _connections[sockN]; + c.gotHello = false; + c.node = _node; // impossible value for a remote + c.sock = sockN; + *uptrN = (void *)&c; + sendHello(sockN); + } + + inline void phyOnTcpClose(ztVsdmInternal::PhySocket *sock,void **uptr) + { + std::lock_guard l(_lock); + _connections.erase(sock); + } + + inline void phyOnTcpData(ztVsdmInternal::PhySocket *sock,void **uptr,void *data,unsigned long len) + { + _connection *const c = (_connection *)*uptr; + if (!c) return; + + std::unique_lock l(_lock); + c->inbuf.append(reinterpret_cast(data),len); + for(;;) { + if (c->gotHello) { + + if (c->inbuf.length() >= 20) { // got message size and header + uint32_t _totalLen; + memcpy(&_totalLen,c->inbuf.data(),4); + const unsigned long totalLen = ntohl(_totalLen); + if ((totalLen > L)||(totalLen < 16)) { // message too small or too large + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + if (c->inbuf.length() >= (4 + totalLen)) { // got full message + + if (!_cryptor.decrypt(reinterpret_cast(const_cast(c->inbuf.data()) + 4),totalLen)) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + uint32_t hdr[4]; + memcpy(hdr,c->inbuf.data() + 4,16); + + const uint64_t objectRev = ((uint64_t)ntohl(hdr[0]) << 32) | (uint64_t)ntohl(hdr[1]); + const unsigned long keyLen = (unsigned long)ntohl(hdr[2]); + unsigned long valueLen = (unsigned long)ntohl(hdr[3]); + + if (objectRev > _rev) + _rev = objectRev; + + uint64_t deletedAt = 0; + if (valueLen == 0xffffffff) { + valueLen = 0; + deletedAt = _rev; + } + if ((keyLen + valueLen + 16 + C::overhead()) > totalLen) { // key and/or value length invalid + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + K k; + if (!S::objectDeserialize(c->inbuf.data() + 16 + 4,keyLen,k)) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + vsdm_entry &e = _m[k]; + if (e.rev < objectRev) { + const bool added = (e.rev == 0); + e.rev = objectRev; + e.deletedAt = deletedAt; + if (e.deletedAt) { + e.v = V(); + } else { + if (!S::objectDeserialize(c->inbuf.data() + 16 + 4 + keyLen,valueLen,e.v)) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + } +#ifdef VSDM_DEBUG + fprintf(stderr,"<< %lu: %s=%s\n",(unsigned long)c->node,k.c_str(),(deletedAt) ? "" : e.v.c_str()); fflush(stderr); +#endif + sendUpdateToAll(sock,c->node,k,e); + + l.unlock(); + try { + if (added) { + _watcher.add(c->node,k,e.v,objectRev); + } else if (deletedAt) { + _watcher.del(c->node,k); + } else { + _watcher.update(c->node,k,e.v,objectRev); + } + } catch ( ... ) {} + l.lock(); + } + + c->inbuf.erase(c->inbuf.begin(),c->inbuf.begin() + totalLen + 4); + + // continue and process more messages in queue, if any + } else { // still waiting on full message + break; + } + } else { // still waiting on message size and header + break; + } + + } else if (c->inbuf.length() >= (24 + C::overhead())) { // got hello header + + if (!_cryptor.decrypt(reinterpret_cast(const_cast(c->inbuf.data())),24 + C::overhead())) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + uint64_t hdr[3]; + memcpy(hdr,c->inbuf.data(),24); + c->inbuf.erase(c->inbuf.begin(),c->inbuf.begin() + 24 + C::overhead()); + + if (htonl(1) != 1) { + hdr[0] = ztVsdmInternal::_swap64(hdr[0]); + hdr[1] = ztVsdmInternal::_swap64(hdr[1]); + hdr[2] = ztVsdmInternal::_swap64(hdr[2]); + } + + if ((hdr[0] == _node)||(hdr[1] != _id)) { // don't connect to self, and don't connect to other map IDs + _connections.erase(sock); + _phy.close(sock,false); + break; + } else { + c->gotHello = true; + c->node = hdr[0]; + + if (hdr[2] > _rev) + _rev = hdr[2]; + + for(typename M::const_iterator i(_m.begin());i!=_m.end();++i) { + if (i->second.rev >= hdr[2]) { + sendUpdate(*c,i->first,i->second); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=%s (new link)\n",(unsigned long)c->node,i->first.c_str(),i->second.v.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + } + + // continue and process more messages in queue, if any + } else { // still waiting on hello header + break; + } + } + } + + inline void phyOnTcpWritable(ztVsdmInternal::PhySocket *sock,void **uptr) + { + std::lock_guard l(_lock); + _connection *c = (_connection *)*uptr; + if (c) { + if (c->outbuf.length() > 0) { + long n = _phy.streamSend(sock,c->outbuf.data(),c->outbuf.length()); + if (n <= 0) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } else if (n == (long)c->outbuf.length()) { + c->outbuf.clear(); + } else { + c->outbuf.erase(c->outbuf.begin(),c->outbuf.begin() + n); + } + } + if (c->outbuf.length() == 0) { + _phy.setNotifyWritable(c->sock,false); + } + } + } + + inline void phyOnDatagram(ztVsdmInternal::PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) {} + inline void phyOnFileDescriptorActivity(ztVsdmInternal::PhySocket *sock,void **uptr,bool readable,bool writable) {} + inline void phyOnUnixAccept(ztVsdmInternal::PhySocket *sockL,ztVsdmInternal::PhySocket *sockN,void **uptrL,void **uptrN) {} + inline void phyOnUnixClose(ztVsdmInternal::PhySocket *sock,void **uptr) {} + inline void phyOnUnixData(ztVsdmInternal::PhySocket *sock,void **uptr,void *data,unsigned long len) {} + inline void phyOnUnixWritable(ztVsdmInternal::PhySocket *sock,void **uptr) {} + + const uint64_t _node; + const uint64_t _id; + uint64_t _rev; + std::unordered_map _peers; + std::unordered_map _connections; + M _m; + mutable std::mutex _lock; + ztVsdmInternal::Phy _phy; + C _cryptor; + W _watcher; + volatile bool _run; + bool _restrictInbound; + std::thread _t; +}; + +#endif diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 5126c5a2..4709b116 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -157,21 +157,6 @@ extern "C" { */ #define ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH 0x0000000000000001ULL -/** - * Maximum number of cluster members (and max member ID plus one) - */ -#define ZT_CLUSTER_MAX_MEMBERS 128 - -/** - * Maximum number of physical ZeroTier addresses a cluster member can report - */ -#define ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES 16 - -/** - * Maximum allowed cluster message length in bytes - */ -#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48) - /** * Maximum value for link quality (min is 0) */ @@ -1104,78 +1089,6 @@ typedef struct unsigned long peerCount; } ZT_PeerList; -/** - * A cluster member's status - */ -typedef struct { - /** - * This cluster member's ID (from 0 to 1-ZT_CLUSTER_MAX_MEMBERS) - */ - unsigned int id; - - /** - * Number of milliseconds since last 'alive' heartbeat message received via cluster backplane address - */ - unsigned int msSinceLastHeartbeat; - - /** - * Non-zero if cluster member is alive - */ - int alive; - - /** - * X, Y, and Z coordinates of this member (if specified, otherwise zero) - * - * What these mean depends on the location scheme being used for - * location-aware clustering. At present this is GeoIP and these - * will be the X, Y, and Z coordinates of the location on a spherical - * approximation of Earth where Earth's core is the origin (in km). - * They don't have to be perfect and need only be comparable with others - * to find shortest path via the standard vector distance formula. - */ - int x,y,z; - - /** - * Cluster member's last reported load - */ - uint64_t load; - - /** - * Number of peers - */ - uint64_t peers; - - /** - * Physical ZeroTier endpoints for this member (where peers are sent when directed here) - */ - struct sockaddr_storage zeroTierPhysicalEndpoints[ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES]; - - /** - * Number of physical ZeroTier endpoints this member is announcing - */ - unsigned int numZeroTierPhysicalEndpoints; -} ZT_ClusterMemberStatus; - -/** - * ZeroTier cluster status - */ -typedef struct { - /** - * My cluster member ID (a record for 'self' is included in member[]) - */ - unsigned int myId; - - /** - * Number of cluster members - */ - unsigned int clusterSize; - - /** - * Cluster member statuses - */ - ZT_ClusterMemberStatus members[ZT_CLUSTER_MAX_MEMBERS]; -} ZT_ClusterStatus; - /** * An instance of a ZeroTier One node (opaque) */ @@ -1765,116 +1678,6 @@ int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t type */ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); -/** - * Initialize cluster operation - * - * This initializes the internal structures and state for cluster operation. - * It takes two function pointers. The first is to a function that can be - * used to send data to cluster peers (mechanism is not defined by Node), - * and the second is to a function that can be used to get the location of - * a physical address in X,Y,Z coordinate space (e.g. as cartesian coordinates - * projected from the center of the Earth). - * - * Send function takes an arbitrary pointer followed by the cluster member ID - * to send data to, a pointer to the data, and the length of the data. The - * maximum message length is ZT_CLUSTER_MAX_MESSAGE_LENGTH (65535). Messages - * must be delivered whole and may be dropped or transposed, though high - * failure rates are undesirable and can cause problems. Validity checking or - * CRC is also not required since the Node validates the authenticity of - * cluster messages using cryptogrphic methods and will silently drop invalid - * messages. - * - * Address to location function is optional and if NULL geo-handoff is not - * enabled (in this case x, y, and z in clusterInit are also unused). It - * takes an arbitrary pointer followed by a physical address and three result - * parameters for x, y, and z. It returns zero on failure or nonzero if these - * three coordinates have been set. Coordinate space is arbitrary and can be - * e.g. coordinates on Earth relative to Earth's center. These can be obtained - * from latitutde and longitude with versions of the Haversine formula. - * - * See: http://stackoverflow.com/questions/1185408/converting-from-longitude-latitude-to-cartesian-coordinates - * - * Neither the send nor the address to location function should block. If the - * address to location function does not have a location for an address, it - * should return zero and then look up the address for future use since it - * will be called again in (typically) 1-3 minutes. - * - * Note that both functions can be called from any thread from which the - * various Node functions are called, and so must be thread safe if multiple - * threads are being used. - * - * @param node Node instance - * @param myId My cluster member ID (less than or equal to ZT_CLUSTER_MAX_MEMBERS) - * @param zeroTierPhysicalEndpoints Preferred physical address(es) for ZeroTier clients to contact this cluster member (for peer redirect) - * @param numZeroTierPhysicalEndpoints Number of physical endpoints in zeroTierPhysicalEndpoints[] (max allowed: 255) - * @param x My cluster member's X location - * @param y My cluster member's Y location - * @param z My cluster member's Z location - * @param sendFunction Function to be called to send data to other cluster members - * @param sendFunctionArg First argument to sendFunction() - * @param addressToLocationFunction Function to be called to get the location of a physical address or NULL to disable geo-handoff - * @param addressToLocationFunctionArg First argument to addressToLocationFunction() - * @return OK or UNSUPPORTED_OPERATION if this Node was not built with cluster support - */ -enum ZT_ResultCode ZT_Node_clusterInit( - ZT_Node *node, - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg); - -/** - * Add a member to this cluster - * - * Calling this without having called clusterInit() will do nothing. - * - * @param node Node instance - * @param memberId Member ID (must be less than or equal to ZT_CLUSTER_MAX_MEMBERS) - * @return OK or error if clustering is disabled, ID invalid, etc. - */ -enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId); - -/** - * Remove a member from this cluster - * - * Calling this without having called clusterInit() will do nothing. - * - * @param node Node instance - * @param memberId Member ID to remove (nothing happens if not present) - */ -void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId); - -/** - * Handle an incoming cluster state message - * - * The message itself contains cluster member IDs, and invalid or badly - * addressed messages will be silently discarded. - * - * Calling this without having called clusterInit() will do nothing. - * - * @param node Node instance - * @param msg Cluster message - * @param len Length of cluster message - */ -void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len); - -/** - * Get the current status of the cluster from this node's point of view - * - * Calling this without clusterInit() or without cluster support will just - * zero out the structure and show a cluster size of zero. - * - * @param node Node instance - * @param cs Cluster status structure to fill with data - */ -void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs); - /** * Set trusted paths * diff --git a/make-mac.mk b/make-mac.mk index de66f49c..5622a41b 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -54,7 +54,7 @@ ifeq ($(ZT_DEBUG),1) # C25519 in particular is almost UNUSABLE in heavy testing without it. node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else - CFLAGS?=-Ofast -fstack-protector-strong + CFLAGS?=-Os -fstack-protector-strong CFLAGS+=$(ARCH_FLAGS) -Wall -Werror -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif diff --git a/node/Node.cpp b/node/Node.cpp index 911c9c4b..8849e0f4 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -503,6 +503,7 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } +/* ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -570,6 +571,7 @@ void Node::clusterStatus(ZT_ClusterStatus *cs) #endif memset(cs,0,sizeof(ZT_ClusterStatus)); } +*/ /****************************************************************************/ /* Node methods used only within node/ */ @@ -998,56 +1000,6 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_clusterInit( - ZT_Node *node, - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg) -{ - try { - return reinterpret_cast(node)->clusterInit(myId,zeroTierPhysicalEndpoints,numZeroTierPhysicalEndpoints,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - -enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId) -{ - try { - return reinterpret_cast(node)->clusterAddMember(memberId); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - -void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId) -{ - try { - reinterpret_cast(node)->clusterRemoveMember(memberId); - } catch ( ... ) {} -} - -void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len) -{ - try { - reinterpret_cast(node)->clusterHandleIncomingMessage(msg,len); - } catch ( ... ) {} -} - -void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs) -{ - try { - reinterpret_cast(node)->clusterStatus(cs); - } catch ( ... ) {} -} - void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count) { try { diff --git a/node/Node.hpp b/node/Node.hpp index 57b5489e..006551fa 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -117,21 +117,6 @@ public: void clearLocalInterfaceAddresses(); int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setNetconfMaster(void *networkControllerInstance); - ZT_ResultCode clusterInit( - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg); - ZT_ResultCode clusterAddMember(unsigned int memberId); - void clusterRemoveMember(unsigned int memberId); - void clusterHandleIncomingMessage(const void *msg,unsigned int len); - void clusterStatus(ZT_ClusterStatus *cs); // Internal functions ------------------------------------------------------ -- cgit v1.2.3 From 64b7d9ef82d73038509b686a46ce5816847089af Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Jun 2017 07:15:46 -0700 Subject: New clustering work. --- attic/DBM.cpp | 243 ++++++++ attic/DBM.hpp | 168 ++++++ ext/kissdb/Makefile | 7 - ext/kissdb/README.md | 69 --- ext/kissdb/SPEC.txt | 62 -- ext/kissdb/kissdb.c | 452 --------------- ext/kissdb/kissdb.h | 173 ------ ext/vsdm/LICENSE.txt | 22 - ext/vsdm/Makefile | 5 - ext/vsdm/README.md | 40 -- ext/vsdm/vsdm-test.cpp | 72 --- ext/vsdm/vsdm.hpp | 1491 ------------------------------------------------ include/ZeroTierOne.h | 56 ++ node/Buffer.hpp | 40 +- node/Constants.hpp | 6 + node/Hashtable.hpp | 7 +- 16 files changed, 481 insertions(+), 2432 deletions(-) create mode 100644 attic/DBM.cpp create mode 100644 attic/DBM.hpp delete mode 100644 ext/kissdb/Makefile delete mode 100644 ext/kissdb/README.md delete mode 100644 ext/kissdb/SPEC.txt delete mode 100644 ext/kissdb/kissdb.c delete mode 100644 ext/kissdb/kissdb.h delete mode 100644 ext/vsdm/LICENSE.txt delete mode 100644 ext/vsdm/Makefile delete mode 100644 ext/vsdm/README.md delete mode 100644 ext/vsdm/vsdm-test.cpp delete mode 100644 ext/vsdm/vsdm.hpp (limited to 'node') diff --git a/attic/DBM.cpp b/attic/DBM.cpp new file mode 100644 index 00000000..54f017e0 --- /dev/null +++ b/attic/DBM.cpp @@ -0,0 +1,243 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#include "DBM.hpp" + +#include "../version.h" + +#include "../node/Salsa20.hpp" +#include "../node/Poly1305.hpp" +#include "../node/SHA512.hpp" + +#include "../osdep/OSUtils.hpp" + +#define ZT_STORED_OBJECT_TYPE__CLUSTER_NODE_STATUS (ZT_STORED_OBJECT__MAX_TYPE_ID + 1) +#define ZT_STORED_OBJECT_TYPE__CLUSTER_DEFINITION (ZT_STORED_OBJECT__MAX_TYPE_ID + 2) + +namespace ZeroTier { + +// We generate the cluster ID from our address and version info since this is +// not at all designed to allow interoperation between versions (or endians) +// in the same cluster. +static inline uint64_t _mkClusterId(const Address &myAddress) +{ + uint64_t x = ZEROTIER_ONE_VERSION_MAJOR; + x <<= 8; + x += ZEROTIER_ONE_VERSION_MINOR; + x <<= 8; + x += ZEROTIER_ONE_VERSION_REVISION; + x <<= 40; + x ^= myAddress.toInt(); +#if __BYTE_ORDER == __BIG_ENDIAN + ++x; +#endif; + return x; +} + +void DBM::onUpdate(uint64_t from,const _MapKey &k,const _MapValue &v,uint64_t rev) +{ + char p[4096]; + char tmp[ZT_DBM_MAX_VALUE_SIZE]; + if (_persistentPath((ZT_StoredObjectType)k.type,k.key,p,sizeof(p))) { + // Reduce unnecessary disk writes + FILE *f = fopen(p,"r"); + if (f) { + long n = (long)fread(tmp,1,sizeof(tmp),f); + fclose(f); + if ((n == (long)v.len)&&(!memcmp(v.data,tmp,n))) + return; + } + + // Write to disk if file has changed or was not already present + f = fopen(p,"w"); + if (f) { + if (fwrite(data,len,1,f) != 1) + fprintf(stderr,"WARNING: error writing to %s (I/O error)" ZT_EOL_S,p); + fclose(f); + if (type == ZT_STORED_OBJECT_IDENTITY_SECRET) + OSUtils::lockDownFile(p,false); + } else { + fprintf(stderr,"WARNING: error writing to %s (cannot open)" ZT_EOL_S,p); + } + } +} + +void DBM::onDelete(uint64_t from,const _MapKey &k) +{ + char p[4096]; + if (_persistentPath((ZT_StoredObjectType)k.type,k.key,p,sizeof(p))) + OSUtils::rm(p); +} + +DBM::_vsdm_cryptor::_vsdm_cryptor(const Identity &secretIdentity) +{ + uint8_t s512[64]; + SHA512::hash(h512,secretIdentity.privateKeyPair().priv.data,ZT_C25519_PRIVATE_KEY_LEN); + memcpy(_key,s512,sizeof(_key)); +} + +void DBM::_vsdm_cryptor::encrypt(void *d,unsigned long l) +{ + if (l >= 24) { // sanity check + uint8_t key[32]; + uint8_t authKey[32]; + uint8_t auth[16]; + + uint8_t *const iv = reinterpret_cast(d) + (l - 16); + Utils::getSecureRandom(iv,16); + memcpy(key,_key,32); + for(unsigned long i=0;i<8;++i) + _key[i] ^= iv[i]; + + Salsa20 s20(key,iv + 8); + memset(authKey,0,32); + s20.crypt12(authKey,authKey,32); + s20.crypt12(d,d,l - 24); + + Poly1305::compute(auth,d,l - 24,authKey); + memcpy(reinterpret_cast(d) + (l - 24),auth,8); + } +} + +bool DBM::_vsdm_cryptor::decrypt(void *d,unsigned long l) +{ + if (l >= 24) { // sanity check + uint8_t key[32]; + uint8_t authKey[32]; + uint8_t auth[16]; + + uint8_t *const iv = reinterpret_cast(d) + (l - 16); + memcpy(key,_key,32); + for(unsigned long i=0;i<8;++i) + _key[i] ^= iv[i]; + + Salsa20 s20(key,iv + 8); + memset(authKey,0,32); + s20.crypt12(authKey,authKey,32); + + Poly1305::compute(auth,d,l - 24,authKey); + if (!Utils::secureEq(reinterpret_cast(d) + (l - 24),auth,8)) + return false; + + s20.crypt12(d,d,l - 24); + + return true; + } + return false; +} + +DBM::DBM(const Identity &secretIdentity,uint64_t clusterMemberId,const std::string &basePath,Node *node) : + _basePath(basePath), + _node(node), + _startTime(OSUtils::now()), + _m(_mkClusterId(secretIdentity.address()),clusterMemberId,false,_vsdm_cryptor(secretIdentity),_vsdm_watcher(this)) +{ +} + +DBM::~DBM() +{ +} + +void DBM::put(const ZT_StoredObjectType type,const uint64_t key,const void *data,unsigned int len) +{ + char p[4096]; + if (_m.put(_MapKey(key,(uint16_t)type),Value(OSUtils::now(),(uint16_t)len,data))) { + if (_persistentPath(type,key,p,sizeof(p))) { + FILE *f = fopen(p,"w"); + if (f) { + if (fwrite(data,len,1,f) != 1) + fprintf(stderr,"WARNING: error writing to %s (I/O error)" ZT_EOL_S,p); + fclose(f); + if (type == ZT_STORED_OBJECT_IDENTITY_SECRET) + OSUtils::lockDownFile(p,false); + } else { + fprintf(stderr,"WARNING: error writing to %s (cannot open)" ZT_EOL_S,p); + } + } + } +} + +bool DBM::get(const ZT_StoredObjectType type,const uint64_t key,Value &value) +{ + char p[4096]; + if (_m.get(_MapKey(key,(uint16_t)type),value)) + return true; + if (_persistentPath(type,key,p,sizeof(p))) { + FILE *f = fopen(p,"r"); + if (f) { + long n = (long)fread(value.data,1,sizeof(value.data),f); + value.len = (n > 0) ? (uint16_t)n : (uint16_t)0; + fclose(f); + value.ts = OSUtils::getLastModified(p); + _m.put(_MapKey(key,(uint16_t)type),value); + return true; + } + } + return false; +} + +void DBM::del(const ZT_StoredObjectType type,const uint64_t key) +{ + char p[4096]; + _m.del(_MapKey(key,(uint16_t)type)); + if (_persistentPath(type,key,p,sizeof(p))) + OSUtils::rm(p); +} + +void DBM::clean() +{ +} + +bool DBM::_persistentPath(const ZT_StoredObjectType type,const uint64_t key,char *p,unsigned int maxlen) +{ + switch(type) { + case ZT_STORED_OBJECT_IDENTITY_PUBLIC: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "identity.public",_basePath.c_str()); + return true; + case ZT_STORED_OBJECT_IDENTITY_SECRET: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "identity.secret",_basePath.c_str()); + return true; + case ZT_STORED_OBJECT_IDENTITY: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "iddb.d" ZT_PATH_SEPARATOR_S "%.10llx",_basePath.c_str(),key); + return true; + case ZT_STORED_OBJECT_NETWORK_CONFIG: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf",_basePath.c_str(),key); + return true; + case ZT_STORED_OBJECT_PLANET: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "planet",_basePath.c_str()); + return true; + case ZT_STORED_OBJECT_MOON: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "moons.d" ZT_PATH_SEPARATOR_S "%.16llx.moon",_basePath.c_str(),key); + return true; + case (ZT_StoredObjectType)ZT_STORED_OBJECT_TYPE__CLUSTER_DEFINITION: + Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "cluster",_basePath.c_str()); + return true; + default: + return false; + } +} + +} // namespace ZeroTier diff --git a/attic/DBM.hpp b/attic/DBM.hpp new file mode 100644 index 00000000..c6d5b8c0 --- /dev/null +++ b/attic/DBM.hpp @@ -0,0 +1,168 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_DBM_HPP___ +#define ZT_DBM_HPP___ + +#include +#include +#include +#include + +#include + +#include "../node/Constants.hpp" +#include "../node/Mutex.hpp" +#include "../node/Utils.hpp" +#include "../node/Identity.hpp" +#include "../node/Peer.hpp" + +#include "../ext/vsdm/vsdm.hpp" + +// The Peer is the largest structure we persist here +#define ZT_DBM_MAX_VALUE_SIZE sizeof(Peer) + +namespace ZeroTier { + +class Node; +class DBM; + +class DBM +{ +public: + ZT_PACKED_STRUCT(struct Value + { + Value(const uint64_t t,const uint16_t l,const void *d) : + ts(t), + l(l) + { + memcpy(data,d,l); + } + uint64_t ts; + uint16_t len; + uint8_t data[ZT_DBM_MAX_VALUE_SIZE]; + }); + +private: + ZT_PACKED_STRUCT(struct _MapKey + { + _MapKey() : obj(0),type(0) {} + _MapKey(const uint16_t t,const uint64_t o) : obj(o),type(t) {} + uint64_t obj; + uint16_t type; + inline bool operator==(const _MapKey &k) const { return ((obj == k.obj)&&(type == k.type)); } + }); + struct _MapHasher + { + inline std::size_t operator()(const _MapKey &k) const { return (std::size_t)((k.obj ^ (k.obj >> 32)) + (uint64_t)k.type); } + }; + + void onUpdate(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev); + void onDelete(uint64_t from,const _MapKey &k); + + class _vsdm_watcher + { + public: + _vsdm_watcher(DBM *p) : _parent(p) {} + inline void add(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev) { _parent->onUpdate(from,k,v,rev); } + inline void update(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev) { _parent->onUpdate(from,k,v,rev); } + inline void del(uint64_t from,const _MapKey &k) { _parent->onDelete(from,k); } + private: + DBM *_parent; + }; + class _vsdm_serializer + { + public: + static inline unsigned long objectSize(const _MapKey &k) { return 10; } + static inline unsigned long objectSize(const Value &v) { return (10 + v.len); } + static inline const char *objectData(const _MapKey &k) { return reinterpret_cast(&k); } + static inline const char *objectData(const Value &v) { return reinterpret_cast(&v); } + static inline bool objectDeserialize(const char *d,unsigned long l,_MapKey &k) + { + if (l == 10) { + memcpy(&k,d,10); + return true; + } + return false; + } + static inline bool objectDeserialize(const char *d,unsigned long l,Value &v) + { + if ((l >= 10)&&(l <= (10 + ZT_DBM_MAX_VALUE_SIZE))) { + memcpy(&v,d,l); + return true; + } + return false; + } + }; + class _vsdm_cryptor + { + public: + _vsdm_cryptor(const Identity &secretIdentity); + static inline unsigned long overhead() { return 24; } + void encrypt(void *d,unsigned long l); + bool decrypt(void *d,unsigned long l); + uint8_t _key[32]; + }; + + typedef vsdm< _MapKey,Value,16384,_vsdm_watcher,_vsdm_serializer,_vsdm_cryptor,_MapHasher > _Map; + + friend class _Map; + +public: + ZT_PACKED_STRUCT(struct ClusterPeerStatus + { + uint64_t startTime; + uint64_t currentTime; + uint64_t clusterPeersConnected; + uint64_t ztPeersConnected; + uint16_t platform; + uint16_t arch; + }); + + DBM(const Identity &secretIdentity,uint64_t clusterMemberId,const std::string &basePath,Node *node); + + ~DBM(); + + void put(const ZT_StoredObjectType type,const uint64_t key,const void *data,unsigned int len); + + bool get(const ZT_StoredObjectType type,const uint64_t key,Value &value); + + void del(const ZT_StoredObjectType type,const uint64_t key); + + void clean(); + +private: + bool DBM::_persistentPath(const ZT_StoredObjectType type,const uint64_t key,char *p,unsigned int maxlen); + + const std::string _basePath; + Node *const _node; + uint64_t _startTime; + _Map _m; +}; + +} // namespace ZeroTier + +#endif diff --git a/ext/kissdb/Makefile b/ext/kissdb/Makefile deleted file mode 100644 index f47372c6..00000000 --- a/ext/kissdb/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# http://creativecommons.org/publicdomain/zero/1.0/ - -all: - gcc -Wall -O2 -DKISSDB_TEST -o kissdb-test kissdb.c - -clean: - rm -f kissdb-test *.o test.db diff --git a/ext/kissdb/README.md b/ext/kissdb/README.md deleted file mode 100644 index ab8a6cff..00000000 --- a/ext/kissdb/README.md +++ /dev/null @@ -1,69 +0,0 @@ -kissdb -====== - -(Keep It) Simple Stupid Database - -KISSDB is about the simplest key/value store you'll ever see, anywhere. -It's written in plain vanilla C using only the standard string and FILE -I/O functions, and should port to just about anything with a disk or -something that acts like one. - -It stores keys and values of fixed length in a stupid-simple file format -based on fixed-size hash tables. If a hash collision occurrs, a new "page" -of hash table is appended to the database. The format is append-only. -There is no delete. Puts that replace an existing value, however, will not -grow the file as they will overwrite the existing entry. - -Hash table size is a space/speed trade-off parameter. Larger hash tables -will reduce collisions and speed things up a bit, at the expense of memory -and disk space. A good size is usually about 1/2 the average number of -entries you expect. - -Features: - - * Tiny, compiles to ~4k on an x86_64 Linux system - * Small memory footprint (only caches hash tables) - * Very space-efficient (on disk) if small hash tables are used - * Makes a decent effort to be robust on power loss - * Pretty respectably fast, especially given its simplicity - * 64-bit, file size limit is 2^64 bytes - * Ports to anything with a C compiler and stdlib/stdio - * Public domain - -Limitations: - - * Fixed-size keys and values, must recreate and copy to change any init size parameter - * Add/update only, no delete - * Iteration is supported but key order is undefined - * No search for subsets of keys/values - * No indexes - * No transactions - * No special recovery features if a database gets corrupted - * No built-in thread-safety (guard it with a mutex in MT code) - * No built-in caching of data (only hash tables are cached for lookup speed) - * No endian-awareness (currently), so big-endian DBs won't read on little-endian machines - -Alternative key/value stores and embedded databases: - - * [MDB](http://symas.com/mdb/) uses mmap() and is very fast (not quite as tiny/simple/portable) - * [CDB](http://cr.yp.to/cdb.html) is also minimal and fast, probably the closest thing to this (but has a 4gb size limit) - * [Kyoto Cabinet](http://fallabs.com/kyotocabinet/) is very fast, full-featured, and modern (license required for commercial use) - * [SQLite](http://www.sqlite.org/) gives you a complete embedded SQL server (public domain, very mature, much larger) - * Others include GDBM, NDBM, Berkeley DB, etc. Use your Googles. :) - -KISSDB is good if you want space-efficient relatively fast write-once/read-many storage -of keys mapped to values. It's not a good choice if you need searches, indexes, delete, -structured storage, or widely varying key/value sizes. It's also probably not a good -choice if you need a long-lived database for critical data, since it lacks recovery -features and is brittle if its internals are modified. It would be better for a cache -of data that can be restored or "re-learned," such as keys, Bitcoin transactions, nodes -on a peer-to-peer network, log analysis results, rendered web pages, session cookies, -auth tokens, etc. - -KISSDB is in the public domain as according to the [Creative Commons Public Domain Dedication](http://creativecommons.org/publicdomain/zero/1.0/). -One reason it was written was the poverty of simple key/value databases with wide open licensing. Even old ones like GDBM have GPL, not LGPL, licenses. - -See comments in kissdb.h for documentation. Makefile can be used to build -a test program on systems with gcc. - -Author: Adam Ierymenko / ZeroTier Networks LLC diff --git a/ext/kissdb/SPEC.txt b/ext/kissdb/SPEC.txt deleted file mode 100644 index 732c4df5..00000000 --- a/ext/kissdb/SPEC.txt +++ /dev/null @@ -1,62 +0,0 @@ ------ - -KISSDB file format (version 2) -Author: Adam Ierymenko - -http://creativecommons.org/publicdomain/zero/1.0/ - ------ - -In keeping with the goal of minimalism the file format is very simple, the -sort of thing that would be given as an example in an introductory course in -data structures. It's a basic hash table that adds additional pages of hash -table entries on collision. - -It consists of a 28 byte header followed by a series of hash tables and data. -All integer values are stored in the native word order of the target -architecture (in the future the code might be fixed to make everything -little-endian if anyone cares about that). - -The header consists of the following fields: - -[0-3] magic numbers: (ASCII) 'K', 'd', 'B', KISSDB_VERSION (currently 2) -[4-11] 64-bit hash table size in entries -[12-19] 64-bit key size in bytes -[20-27] 64-bit value size in bytes - -Hash tables are arrays of [hash table size + 1] 64-bit integers. The extra -entry, if nonzero, is the offset in the file of the next hash table, forming -a linked list of hash tables across the file. - -Immediately following the header, the first hash table will be written when -the first key/value is added. The algorithm for adding new entries is as -follows: - -(1) The key is hashed using a 64-bit variant of the DJB2 hash function, and - this is taken modulo hash table size to get a bucket number. -(2) Hash tables are checked in order, starting with the first hash table, - until a zero (empty) bucket is found. If one is found, skip to step (4). -(3) If no empty buckets are found in any hash table, a new table is appended - to the file and the final pointer in the previous hash table is set to - its offset. (In the code the update of the next hash table pointer in - the previous hash table happens last, after the whole write is complete, - to avoid corruption on power loss.) -(4) The key and value are appended, in order with no additional meta-data, - to the database file. Before appending the offset in the file stream - where they will be stored is saved. After appending, this offset is - written to the empty hash table bucket we chose in steps 2/3. Hash table - updates happen last to avoid corruption if the write does not complete. - -Lookup of a key/value pair occurs as follows: - -(1) The key is hashed and taken modulo hash table size to get a bucket - number. -(2) If this bucket's entry in the hash table is nonzero, the key at the - offset specified by this bucket is compared to the key being looked up. - If they are equal, the value is read and returned. -(3) If the keys are not equal, the next hash table is checked and step (2) - is repeated. If an empty bucket is encountered or if we run out of hash - tables, the key was not found. - -To update an existing value, its location is looked up and the value portion -of the entry is rewritten. diff --git a/ext/kissdb/kissdb.c b/ext/kissdb/kissdb.c deleted file mode 100644 index 6b275686..00000000 --- a/ext/kissdb/kissdb.c +++ /dev/null @@ -1,452 +0,0 @@ -/* (Keep It) Simple Stupid Database - * - * Written by Adam Ierymenko - * KISSDB is in the public domain and is distributed with NO WARRANTY. - * - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -/* Compile with KISSDB_TEST to build as a test program. */ - -/* Note: big-endian systems will need changes to implement byte swapping - * on hash table file I/O. Or you could just use it as-is if you don't care - * that your database files will be unreadable on little-endian systems. */ - -#define _FILE_OFFSET_BITS 64 - -#include "kissdb.h" - -#include -#include -#include - -#ifdef _WIN32 -#define fseeko _fseeki64 -#define ftello _ftelli64 -#endif - -#define KISSDB_HEADER_SIZE ((sizeof(uint64_t) * 3) + 4) - -/* djb2 hash function */ -static uint64_t KISSDB_hash(const void *b,unsigned long len) -{ - unsigned long i; - uint64_t hash = 5381; - for(i=0;if = (FILE *)0; - fopen_s(&db->f,path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); -#else - db->f = fopen(path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); -#endif - if (!db->f) { - if (mode == KISSDB_OPEN_MODE_RWCREAT) { -#ifdef _WIN32 - db->f = (FILE *)0; - fopen_s(&db->f,path,"w+b"); -#else - db->f = fopen(path,"w+b"); -#endif - } - if (!db->f) - return KISSDB_ERROR_IO; - } - - if (fseeko(db->f,0,SEEK_END)) { - fclose(db->f); - return KISSDB_ERROR_IO; - } - if (ftello(db->f) < KISSDB_HEADER_SIZE) { - /* write header if not already present */ - if ((hash_table_size)&&(key_size)&&(value_size)) { - if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp2[0] = 'K'; tmp2[1] = 'd'; tmp2[2] = 'B'; tmp2[3] = KISSDB_VERSION; - if (fwrite(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp = hash_table_size; - if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp = key_size; - if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp = value_size; - if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - fflush(db->f); - } else { - fclose(db->f); - return KISSDB_ERROR_INVALID_PARAMETERS; - } - } else { - if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } - if (fread(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if ((tmp2[0] != 'K')||(tmp2[1] != 'd')||(tmp2[2] != 'B')||(tmp2[3] != KISSDB_VERSION)) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if (!tmp) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - hash_table_size = (unsigned long)tmp; - if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if (!tmp) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - key_size = (unsigned long)tmp; - if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if (!tmp) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - value_size = (unsigned long)tmp; - } - - db->hash_table_size = hash_table_size; - db->key_size = key_size; - db->value_size = value_size; - db->hash_table_size_bytes = sizeof(uint64_t) * (hash_table_size + 1); /* [hash_table_size] == next table */ - - httmp = malloc(db->hash_table_size_bytes); - if (!httmp) { - fclose(db->f); - return KISSDB_ERROR_MALLOC; - } - db->num_hash_tables = 0; - db->hash_tables = (uint64_t *)0; - while (fread(httmp,db->hash_table_size_bytes,1,db->f) == 1) { - hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); - if (!hash_tables_rea) { - KISSDB_close(db); - free(httmp); - return KISSDB_ERROR_MALLOC; - } - db->hash_tables = hash_tables_rea; - - memcpy(((uint8_t *)db->hash_tables) + (db->hash_table_size_bytes * db->num_hash_tables),httmp,db->hash_table_size_bytes); - ++db->num_hash_tables; - if (httmp[db->hash_table_size]) { - if (fseeko(db->f,httmp[db->hash_table_size],SEEK_SET)) { - KISSDB_close(db); - free(httmp); - return KISSDB_ERROR_IO; - } - } else break; - } - free(httmp); - - return 0; -} - -void KISSDB_close(KISSDB *db) -{ - if (db->hash_tables) - free(db->hash_tables); - if (db->f) - fclose(db->f); - memset(db,0,sizeof(KISSDB)); -} - -int KISSDB_get(KISSDB *db,const void *key,void *vbuf) -{ - uint8_t tmp[4096]; - const uint8_t *kptr; - unsigned long klen,i; - uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; - uint64_t offset; - uint64_t *cur_hash_table; - long n; - - cur_hash_table = db->hash_tables; - for(i=0;inum_hash_tables;++i) { - offset = cur_hash_table[hash]; - if (offset) { - if (fseeko(db->f,offset,SEEK_SET)) - return KISSDB_ERROR_IO; - - kptr = (const uint8_t *)key; - klen = db->key_size; - while (klen) { - n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); - if (n > 0) { - if (memcmp(kptr,tmp,n)) - goto get_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long)n; - } else return 1; /* not found */ - } - - if (fread(vbuf,db->value_size,1,db->f) == 1) - return 0; /* success */ - else return KISSDB_ERROR_IO; - } else return 1; /* not found */ -get_no_match_next_hash_table: - cur_hash_table += db->hash_table_size + 1; - } - - return 1; /* not found */ -} - -int KISSDB_put(KISSDB *db,const void *key,const void *value) -{ - uint8_t tmp[4096]; - const uint8_t *kptr; - unsigned long klen,i; - uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; - uint64_t offset; - uint64_t htoffset,lasthtoffset; - uint64_t endoffset; - uint64_t *cur_hash_table; - uint64_t *hash_tables_rea; - long n; - - lasthtoffset = htoffset = KISSDB_HEADER_SIZE; - cur_hash_table = db->hash_tables; - for(i=0;inum_hash_tables;++i) { - offset = cur_hash_table[hash]; - if (offset) { - /* rewrite if already exists */ - if (fseeko(db->f,offset,SEEK_SET)) - return KISSDB_ERROR_IO; - - kptr = (const uint8_t *)key; - klen = db->key_size; - while (klen) { - n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); - if (n > 0) { - if (memcmp(kptr,tmp,n)) - goto put_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long)n; - } - } - - /* C99 spec demands seek after fread(), required for Windows */ - fseeko(db->f,0,SEEK_CUR); - - if (fwrite(value,db->value_size,1,db->f) == 1) { - fflush(db->f); - return 0; /* success */ - } else return KISSDB_ERROR_IO; - } else { - /* add if an empty hash table slot is discovered */ - if (fseeko(db->f,0,SEEK_END)) - return KISSDB_ERROR_IO; - endoffset = ftello(db->f); - - if (fwrite(key,db->key_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - if (fwrite(value,db->value_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - - if (fseeko(db->f,htoffset + (sizeof(uint64_t) * hash),SEEK_SET)) - return KISSDB_ERROR_IO; - if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) - return KISSDB_ERROR_IO; - cur_hash_table[hash] = endoffset; - - fflush(db->f); - - return 0; /* success */ - } -put_no_match_next_hash_table: - lasthtoffset = htoffset; - htoffset = cur_hash_table[db->hash_table_size]; - cur_hash_table += (db->hash_table_size + 1); - } - - /* if no existing slots, add a new page of hash table entries */ - if (fseeko(db->f,0,SEEK_END)) - return KISSDB_ERROR_IO; - endoffset = ftello(db->f); - - hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); - if (!hash_tables_rea) - return KISSDB_ERROR_MALLOC; - db->hash_tables = hash_tables_rea; - cur_hash_table = &(db->hash_tables[(db->hash_table_size + 1) * db->num_hash_tables]); - memset(cur_hash_table,0,db->hash_table_size_bytes); - - cur_hash_table[hash] = endoffset + db->hash_table_size_bytes; /* where new entry will go */ - - if (fwrite(cur_hash_table,db->hash_table_size_bytes,1,db->f) != 1) - return KISSDB_ERROR_IO; - - if (fwrite(key,db->key_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - if (fwrite(value,db->value_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - - if (db->num_hash_tables) { - if (fseeko(db->f,lasthtoffset + (sizeof(uint64_t) * db->hash_table_size),SEEK_SET)) - return KISSDB_ERROR_IO; - if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) - return KISSDB_ERROR_IO; - db->hash_tables[((db->hash_table_size + 1) * (db->num_hash_tables - 1)) + db->hash_table_size] = endoffset; - } - - ++db->num_hash_tables; - - fflush(db->f); - - return 0; /* success */ -} - -void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi) -{ - dbi->db = db; - dbi->h_no = 0; - dbi->h_idx = 0; -} - -int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf) -{ - uint64_t offset; - - if ((dbi->h_no < dbi->db->num_hash_tables)&&(dbi->h_idx < dbi->db->hash_table_size)) { - while (!(offset = dbi->db->hash_tables[((dbi->db->hash_table_size + 1) * dbi->h_no) + dbi->h_idx])) { - if (++dbi->h_idx >= dbi->db->hash_table_size) { - dbi->h_idx = 0; - if (++dbi->h_no >= dbi->db->num_hash_tables) - return 0; - } - } - if (fseeko(dbi->db->f,offset,SEEK_SET)) - return KISSDB_ERROR_IO; - if (fread(kbuf,dbi->db->key_size,1,dbi->db->f) != 1) - return KISSDB_ERROR_IO; - if (fread(vbuf,dbi->db->value_size,1,dbi->db->f) != 1) - return KISSDB_ERROR_IO; - if (++dbi->h_idx >= dbi->db->hash_table_size) { - dbi->h_idx = 0; - ++dbi->h_no; - } - return 1; - } - - return 0; -} - -#ifdef KISSDB_TEST - -#include - -int main(int argc,char **argv) -{ - uint64_t i,j; - uint64_t v[8]; - KISSDB db; - KISSDB_Iterator dbi; - char got_all_values[10000]; - int q; - - printf("Opening new empty database test.db...\n"); - - if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RWREPLACE,1024,8,sizeof(v))) { - printf("KISSDB_open failed\n"); - return 1; - } - - printf("Adding and then re-getting 10000 64-byte values...\n"); - - for(i=0;i<10000;++i) { - for(j=0;j<8;++j) - v[j] = i; - if (KISSDB_put(&db,&i,v)) { - printf("KISSDB_put failed (%"PRIu64")\n",i); - return 1; - } - memset(v,0,sizeof(v)); - if ((q = KISSDB_get(&db,&i,v))) { - printf("KISSDB_get (1) failed (%"PRIu64") (%d)\n",i,q); - return 1; - } - for(j=0;j<8;++j) { - if (v[j] != i) { - printf("KISSDB_get (1) failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - } - - printf("Getting 10000 64-byte values...\n"); - - for(i=0;i<10000;++i) { - if ((q = KISSDB_get(&db,&i,v))) { - printf("KISSDB_get (2) failed (%"PRIu64") (%d)\n",i,q); - return 1; - } - for(j=0;j<8;++j) { - if (v[j] != i) { - printf("KISSDB_get (2) failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - } - - printf("Closing and re-opening database in read-only mode...\n"); - - KISSDB_close(&db); - - if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RDONLY,1024,8,sizeof(v))) { - printf("KISSDB_open failed\n"); - return 1; - } - - printf("Getting 10000 64-byte values...\n"); - - for(i=0;i<10000;++i) { - if ((q = KISSDB_get(&db,&i,v))) { - printf("KISSDB_get (3) failed (%"PRIu64") (%d)\n",i,q); - return 1; - } - for(j=0;j<8;++j) { - if (v[j] != i) { - printf("KISSDB_get (3) failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - } - - printf("Iterator test...\n"); - - KISSDB_Iterator_init(&db,&dbi); - i = 0xdeadbeef; - memset(got_all_values,0,sizeof(got_all_values)); - while (KISSDB_Iterator_next(&dbi,&i,&v) > 0) { - if (i < 10000) - got_all_values[i] = 1; - else { - printf("KISSDB_Iterator_next failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - for(i=0;i<10000;++i) { - if (!got_all_values[i]) { - printf("KISSDB_Iterator failed, missing value index %"PRIu64"\n",i); - return 1; - } - } - - KISSDB_close(&db); - - printf("All tests OK!\n"); - - return 0; -} - -#endif diff --git a/ext/kissdb/kissdb.h b/ext/kissdb/kissdb.h deleted file mode 100644 index 926906b0..00000000 --- a/ext/kissdb/kissdb.h +++ /dev/null @@ -1,173 +0,0 @@ -/* (Keep It) Simple Stupid Database - * - * Written by Adam Ierymenko - * KISSDB is in the public domain and is distributed with NO WARRANTY. - * - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -#ifndef ___KISSDB_H -#define ___KISSDB_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Version: 2 - * - * This is the file format identifier, and changes any time the file - * format changes. The code version will be this dot something, and can - * be seen in tags in the git repository. - */ -#define KISSDB_VERSION 2 - -/** - * KISSDB database state - * - * These fields can be read by a user, e.g. to look up key_size and - * value_size, but should never be changed. - */ -typedef struct { - unsigned long hash_table_size; - unsigned long key_size; - unsigned long value_size; - unsigned long hash_table_size_bytes; - unsigned long num_hash_tables; - uint64_t *hash_tables; - FILE *f; -} KISSDB; - -/** - * I/O error or file not found - */ -#define KISSDB_ERROR_IO -1 - -/** - * Out of memory - */ -#define KISSDB_ERROR_MALLOC -2 - -/** - * Invalid paramters (e.g. missing _size paramters on init to create database) - */ -#define KISSDB_ERROR_INVALID_PARAMETERS -3 - -/** - * Database file appears corrupt - */ -#define KISSDB_ERROR_CORRUPT_DBFILE -4 - -/** - * Open mode: read only - */ -#define KISSDB_OPEN_MODE_RDONLY 1 - -/** - * Open mode: read/write - */ -#define KISSDB_OPEN_MODE_RDWR 2 - -/** - * Open mode: read/write, create if doesn't exist - */ -#define KISSDB_OPEN_MODE_RWCREAT 3 - -/** - * Open mode: truncate database, open for reading and writing - */ -#define KISSDB_OPEN_MODE_RWREPLACE 4 - -/** - * Open database - * - * The three _size parameters must be specified if the database could - * be created or re-created. Otherwise an error will occur. If the - * database already exists, these parameters are ignored and are read - * from the database. You can check the struture afterwords to see what - * they were. - * - * @param db Database struct - * @param path Path to file - * @param mode One of the KISSDB_OPEN_MODE constants - * @param hash_table_size Size of hash table in 64-bit entries (must be >0) - * @param key_size Size of keys in bytes - * @param value_size Size of values in bytes - * @return 0 on success, nonzero on error - */ -extern int KISSDB_open( - KISSDB *db, - const char *path, - int mode, - unsigned long hash_table_size, - unsigned long key_size, - unsigned long value_size); - -/** - * Close database - * - * @param db Database struct - */ -extern void KISSDB_close(KISSDB *db); - -/** - * Get an entry - * - * @param db Database struct - * @param key Key (key_size bytes) - * @param vbuf Value buffer (value_size bytes capacity) - * @return -1 on I/O error, 0 on success, 1 on not found - */ -extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); - -/** - * Put an entry (overwriting it if it already exists) - * - * In the already-exists case the size of the database file does not - * change. - * - * @param db Database struct - * @param key Key (key_size bytes) - * @param value Value (value_size bytes) - * @return -1 on I/O error, 0 on success - */ -extern int KISSDB_put(KISSDB *db,const void *key,const void *value); - -/** - * Cursor used for iterating over all entries in database - */ -typedef struct { - KISSDB *db; - unsigned long h_no; - unsigned long h_idx; -} KISSDB_Iterator; - -/** - * Initialize an iterator - * - * @param db Database struct - * @param i Iterator to initialize - */ -extern void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi); - -/** - * Get the next entry - * - * The order of entries returned by iterator is undefined. It depends on - * how keys hash. - * - * @param Database iterator - * @param kbuf Buffer to fill with next key (key_size bytes) - * @param vbuf Buffer to fill with next value (value_size bytes) - * @return 0 if there are no more entries, negative on error, positive if an kbuf/vbuf have been filled - */ -extern int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf); - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/ext/vsdm/LICENSE.txt b/ext/vsdm/LICENSE.txt deleted file mode 100644 index 00b65473..00000000 --- a/ext/vsdm/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -MIT LICENSE - -Copyright 2017 ZeroTier, Inc. -https://www.zerotier.com/ - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/ext/vsdm/Makefile b/ext/vsdm/Makefile deleted file mode 100644 index 828d6edc..00000000 --- a/ext/vsdm/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -all: - c++ -Os -std=c++11 -o vsdm-test vsdm-test.cpp - -clean: - rm -f vsdm-test *.o *.dSYM diff --git a/ext/vsdm/README.md b/ext/vsdm/README.md deleted file mode 100644 index 03354be6..00000000 --- a/ext/vsdm/README.md +++ /dev/null @@ -1,40 +0,0 @@ -VSDM: Very Simple Distributed Map -====== - -VSDM is a super-minimal replicated in-memory associative container. Its advantages are small code size, small footprint, simplicity, and lack of dependencies. - -VSDM uses a rumor mill replication algorithm that provides fast best-effort replication. If connectivity is stable this results in eventual consistency, but data loss or regression can occur under split brain conditions. This class is not recommended for data that is intolerant of loss or regression. Its ideal use case is a distributed cache for small data objects or a distributed database for ephemeral data. - -Transport is via TCP and can optionally be encrypted if one specifies a cryptor class (see below). The transport protocol does not implement any features for versioning or backward compatibility and changes to key/value type, cryptor, or other relevant parameters will render it incompatible. Again this is designed for simple app-embedded use cases. Use something more feature complete if you need to support different versions across the network. - -Each node maintains a 64-bit monotonically increasing revision counter that starts at zero. When a node connects to another node it sends this revision counter and the other party will send all updates with revision numbers greater than or equal to it. When a node receives an update it replaces the entry it has if the revision counter is higher and also sets its own revision counter to the new counter if it is higher. It then re-transmits the update if a replace event occurred (if the new update was newer). This is the "rumor mill" part: each node retransmits all changes to all other connected nodes (excluding the source). - -VSDM nodes can be connected according to any arbitrary connectivity graph. A more fully connected graph trades increased bandwidth consumption (due to redundant messages) for decreased likelihood of data loss or split brain conditions. - -The VSDM class is thread safe. It launches a single background thread to handle network I/O and periodic cleanup. Deleted keys are purged from memory after a period of time to allow time for propagation and possible re-propagation. - -## Template parameters - -VSDM supports a number of template parameters for customization. The only required parameters are K and V, the key and value types. The default serializer allows these to be one of the *uintX_t* stdint numeric types or *std::string*. Specify a different serializer for anything else. - -Here are the other parameters after K and V (in order): - - * **L**: Maximum message length, which also limits the max size of the combined key and value for a given entry. This imposes a sanity limit to prevent memory exhaustion. Default is 131072. Absolute max is UINT32_MAX - 4 (4294967291). - * **W**: Watcher function type. Default is `vsdm_watcher_noop` which does nothing. The watcher function (or function object) is passed into the constructor and receives notifications of remotely initiated changes to the map's contents. (Local changes via set() or del() do not trigger the watcher). The watcher must have the following methods: - * `void add(uint64_t remoteNodeId,const K &key,const V &value,uint64_t revision)` - * `void update(uint64_t remoteNodeId,const K &key,const V &value,uint64_t revision)` - * `void del(uint64_t remoteNodeId,const K &key)` - * **S**: Serializer class, which must contain *static* methods for serializing and deserializing keys and values. The following must be present for both key and value types: - * `unsigned long objectSize(const [K|V] &)` - * `const char *objectData(const [K|V] &)` - * `bool objectDeserialize(const char *,unsigned long,[K|V] &)` (false return means object was invalid and causes disconnect) - * **C**: Cryptor type to encrypt transport, default is `vsdm_cryptor_noop` (no encryption). It is also passed into the constructor for internal initialization. This must implement a static method `static unsigned long overhead()` that returns the *constant* per-message overhead for things like IV and MAC, and two methods to encrypt and decrypt the payload in place. The vsdm class will add space for overhead at the *end* of the message. Encrypt/decrypt mathods have these signatures: - * `void encrypt(void *,unsigned long)` - * `bool decrypt(void *,unsigned long)` (false return means invalid MAC and causes disconnect) - * **M**: Map type for underlying storage. Default is `std::unordered_map` with default STL hashers. Substitute a different container if you need to deal with non-hashable keys or need a sorted map. Containers supporting duplicate keys should not be used as the replication algorithm will not function properly. - -## License - -(c)2017 ZeroTier, Inc. (MIT license) - -Written by [Adam Ierymenko](https://github.com/adamierymenko) diff --git a/ext/vsdm/vsdm-test.cpp b/ext/vsdm/vsdm-test.cpp deleted file mode 100644 index 13c49afe..00000000 --- a/ext/vsdm/vsdm-test.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include -#include -#include - -#include - -#define VSDM_DEBUG 1 - -#include "vsdm.hpp" - -int main(int argc,char **argv) -{ - if (argc < 4) { - printf("Usage: vsdm-test [//]\n"); - return 0; - } - - uint64_t id = (uint64_t)strtoull(argv[1],(char **)0,10); - uint64_t node = (uint64_t)strtoull(argv[2],(char **)0,10); - int port = (int)strtol(argv[3],(char **)0,10); - - struct sockaddr_in sa; - memset(&sa,0,sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_port = htons((uint16_t)port); - - vsdm m(id,node,false); - m.listen((const struct sockaddr *)&sa); - - for(int i=4;i - * License: MIT - */ - -#ifndef ZT_VSDM_HPP__ -#define ZT_VSDM_HPP__ - -#include -#include -#include - -#if defined(_WIN32) || defined(_WIN64) - -#include -#include -#include - -#define ZT_PHY_SOCKFD_TYPE SOCKET -#define ZT_PHY_SOCKFD_NULL (INVALID_SOCKET) -#define ZT_PHY_SOCKFD_VALID(s) ((s) != INVALID_SOCKET) -#define ZT_PHY_CLOSE_SOCKET(s) ::closesocket(s) -#define ZT_PHY_MAX_SOCKETS (FD_SETSIZE) -#define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS -#define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage - -#else // not Windows - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define ZT_PHY_SOCKFD_TYPE int -#define ZT_PHY_SOCKFD_NULL (-1) -#define ZT_PHY_SOCKFD_VALID(s) ((s) > -1) -#define ZT_PHY_CLOSE_SOCKET(s) ::close(s) -#define ZT_PHY_MAX_SOCKETS (FD_SETSIZE) -#define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS -#define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage - -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/*********************************************************************************************************/ - -namespace ztVsdmInternal { - -/* This is the Phy<> adapter implementation for selected sockets from ZeroTier One. - * It should build and run out of the box on Windows and most *nix systems. Parts - * not used by VSDM have been removed. */ - -typedef void PhySocket; - -template -class Phy -{ -private: - HANDLER_PTR_TYPE _handler; - - enum PhySocketType - { - ZT_PHY_SOCKET_CLOSED = 0x00, // socket is closed, will be removed on next poll() - ZT_PHY_SOCKET_TCP_OUT_PENDING = 0x01, - ZT_PHY_SOCKET_TCP_OUT_CONNECTED = 0x02, - ZT_PHY_SOCKET_TCP_IN = 0x03, - ZT_PHY_SOCKET_TCP_LISTEN = 0x04, - ZT_PHY_SOCKET_UDP = 0x05, - ZT_PHY_SOCKET_FD = 0x06, - ZT_PHY_SOCKET_UNIX_IN = 0x07, - ZT_PHY_SOCKET_UNIX_LISTEN = 0x08 - }; - - struct PhySocketImpl - { - PhySocketType type; - ZT_PHY_SOCKFD_TYPE sock; - void *uptr; // user-settable pointer - ZT_PHY_SOCKADDR_STORAGE_TYPE saddr; // remote for TCP_OUT and TCP_IN, local for TCP_LISTEN, RAW, and UDP - }; - - std::list _socks; - fd_set _readfds; - fd_set _writefds; -#if defined(_WIN32) || defined(_WIN64) - fd_set _exceptfds; -#endif - long _nfds; - - ZT_PHY_SOCKFD_TYPE _whackReceiveSocket; - ZT_PHY_SOCKFD_TYPE _whackSendSocket; - - bool _noDelay; - bool _noCheck; - -public: - /** - * @param handler Pointer of type HANDLER_PTR_TYPE to handler - * @param noDelay If true, disable TCP NAGLE algorithm on TCP sockets - * @param noCheck If true, attempt to set UDP SO_NO_CHECK option to disable sending checksums - */ - Phy(HANDLER_PTR_TYPE handler,bool noDelay,bool noCheck) : - _handler(handler) - { - FD_ZERO(&_readfds); - FD_ZERO(&_writefds); - -#if defined(_WIN32) || defined(_WIN64) - FD_ZERO(&_exceptfds); - - SOCKET pipes[2]; - { // hack copied from StackOverflow, behaves a bit like pipe() on *nix systems - struct sockaddr_in inaddr; - struct sockaddr addr; - SOCKET lst=::socket(AF_INET, SOCK_STREAM,IPPROTO_TCP); - if (lst == INVALID_SOCKET) - throw std::runtime_error("unable to create pipes for select() abort"); - memset(&inaddr, 0, sizeof(inaddr)); - memset(&addr, 0, sizeof(addr)); - inaddr.sin_family = AF_INET; - inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - inaddr.sin_port = 0; - int yes=1; - setsockopt(lst,SOL_SOCKET,SO_REUSEADDR,(char*)&yes,sizeof(yes)); - bind(lst,(struct sockaddr *)&inaddr,sizeof(inaddr)); - listen(lst,1); - int len=sizeof(inaddr); - getsockname(lst, &addr,&len); - pipes[0]=::socket(AF_INET, SOCK_STREAM,0); - if (pipes[0] == INVALID_SOCKET) - throw std::runtime_error("unable to create pipes for select() abort"); - connect(pipes[0],&addr,len); - pipes[1]=accept(lst,0,0); - closesocket(lst); - } -#else // not Windows - int pipes[2]; - if (::pipe(pipes)) - throw std::runtime_error("unable to create pipes for select() abort"); -#endif // Windows or not - - _nfds = (pipes[0] > pipes[1]) ? (long)pipes[0] : (long)pipes[1]; - _whackReceiveSocket = pipes[0]; - _whackSendSocket = pipes[1]; - _noDelay = noDelay; - _noCheck = noCheck; - } - - ~Phy() - { - for(typename std::list::const_iterator s(_socks.begin());s!=_socks.end();++s) { - if (s->type != ZT_PHY_SOCKET_CLOSED) - this->close((PhySocket *)&(*s),true); - } - ZT_PHY_CLOSE_SOCKET(_whackReceiveSocket); - ZT_PHY_CLOSE_SOCKET(_whackSendSocket); - } - - /** - * @param s Socket object - * @return Underlying OS-type (usually int or long) file descriptor associated with object - */ - static inline ZT_PHY_SOCKFD_TYPE getDescriptor(PhySocket *s) throw() { return reinterpret_cast(s)->sock; } - - /** - * @param s Socket object - * @return Pointer to user object - */ - static inline void** getuptr(PhySocket *s) throw() { return &(reinterpret_cast(s)->uptr); } - - /** - * Cause poll() to stop waiting immediately - * - * This can be used to reset the polling loop after changes that require - * attention, or to shut down a background thread that is waiting, etc. - */ - inline void whack() - { -#if defined(_WIN32) || defined(_WIN64) - ::send(_whackSendSocket,(const char *)this,1,0); -#else - (void)(::write(_whackSendSocket,(PhySocket *)this,1)); -#endif - } - - /** - * @return Number of open sockets - */ - inline unsigned long count() const throw() { return _socks.size(); } - - /** - * @return Maximum number of sockets allowed - */ - inline unsigned long maxCount() const throw() { return ZT_PHY_MAX_SOCKETS; } - - /** - * Bind a local listen socket to listen for new TCP connections - * - * @param localAddress Local address and port - * @param uptr Initial value of uptr for new socket (default: NULL) - * @return Socket or NULL on failure to bind - */ - inline PhySocket *tcpListen(const struct sockaddr *localAddress,void *uptr = (void *)0) - { - if (_socks.size() >= ZT_PHY_MAX_SOCKETS) - return (PhySocket *)0; - - ZT_PHY_SOCKFD_TYPE s = ::socket(localAddress->sa_family,SOCK_STREAM,0); - if (!ZT_PHY_SOCKFD_VALID(s)) - return (PhySocket *)0; - -#if defined(_WIN32) || defined(_WIN64) - { - BOOL f; - f = TRUE; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(const char *)&f,sizeof(f)); - f = TRUE; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(const char *)&f,sizeof(f)); - f = (_noDelay ? TRUE : FALSE); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); - u_long iMode=1; - ioctlsocket(s,FIONBIO,&iMode); - } -#else - { - int f; - f = 1; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f)); - f = 1; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f)); - f = (_noDelay ? 1 : 0); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); - fcntl(s,F_SETFL,O_NONBLOCK); - } -#endif - - if (::bind(s,localAddress,(localAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in))) { - ZT_PHY_CLOSE_SOCKET(s); - return (PhySocket *)0; - } - - if (::listen(s,1024)) { - ZT_PHY_CLOSE_SOCKET(s); - return (PhySocket *)0; - } - - try { - _socks.push_back(PhySocketImpl()); - } catch ( ... ) { - ZT_PHY_CLOSE_SOCKET(s); - return (PhySocket *)0; - } - PhySocketImpl &sws = _socks.back(); - - if ((long)s > _nfds) - _nfds = (long)s; - FD_SET(s,&_readfds); - sws.type = ZT_PHY_SOCKET_TCP_LISTEN; - sws.sock = s; - sws.uptr = uptr; - memset(&(sws.saddr),0,sizeof(struct sockaddr_storage)); - memcpy(&(sws.saddr),localAddress,(localAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); - - return (PhySocket *)&sws; - } - - /** - * Start a non-blocking connect; CONNECT handler is called on success or failure - * - * A return value of NULL indicates a synchronous failure such as a - * failure to open a socket. The TCP connection handler is not called - * in this case. - * - * It is possible on some platforms for an "instant connect" to occur, - * such as when connecting to a loopback address. In this case, the - * 'connected' result parameter will be set to 'true' and if the - * 'callConnectHandler' flag is true (the default) the TCP connect - * handler will be called before the function returns. - * - * These semantics can be a bit confusing, but they're less so than - * the underlying semantics of asynchronous TCP connect. - * - * @param remoteAddress Remote address - * @param connected Result parameter: set to whether an "instant connect" has occurred (true if yes) - * @param uptr Initial value of uptr for new socket (default: NULL) - * @param callConnectHandler If true, call TCP connect handler even if result is known before function exit (default: true) - * @return New socket or NULL on failure - */ - inline PhySocket *tcpConnect(const struct sockaddr *remoteAddress,bool &connected,void *uptr = (void *)0,bool callConnectHandler = true) - { - if (_socks.size() >= ZT_PHY_MAX_SOCKETS) - return (PhySocket *)0; - - ZT_PHY_SOCKFD_TYPE s = ::socket(remoteAddress->sa_family,SOCK_STREAM,0); - if (!ZT_PHY_SOCKFD_VALID(s)) { - connected = false; - return (PhySocket *)0; - } - -#if defined(_WIN32) || defined(_WIN64) - { - BOOL f; - if (remoteAddress->sa_family == AF_INET6) { f = TRUE; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(const char *)&f,sizeof(f)); } - f = TRUE; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(const char *)&f,sizeof(f)); - f = (_noDelay ? TRUE : FALSE); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); - u_long iMode=1; - ioctlsocket(s,FIONBIO,&iMode); - } -#else - { - int f; - if (remoteAddress->sa_family == AF_INET6) { f = 1; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f)); } - f = 1; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f)); - f = (_noDelay ? 1 : 0); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); - fcntl(s,F_SETFL,O_NONBLOCK); - } -#endif - - connected = true; - if (::connect(s,remoteAddress,(remoteAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in))) { - connected = false; -#if defined(_WIN32) || defined(_WIN64) - if (WSAGetLastError() != WSAEWOULDBLOCK) { -#else - if (errno != EINPROGRESS) { -#endif - ZT_PHY_CLOSE_SOCKET(s); - return (PhySocket *)0; - } // else connection is proceeding asynchronously... - } - - try { - _socks.push_back(PhySocketImpl()); - } catch ( ... ) { - ZT_PHY_CLOSE_SOCKET(s); - return (PhySocket *)0; - } - PhySocketImpl &sws = _socks.back(); - - if ((long)s > _nfds) - _nfds = (long)s; - if (connected) { - FD_SET(s,&_readfds); - sws.type = ZT_PHY_SOCKET_TCP_OUT_CONNECTED; - } else { - FD_SET(s,&_writefds); -#if defined(_WIN32) || defined(_WIN64) - FD_SET(s,&_exceptfds); -#endif - sws.type = ZT_PHY_SOCKET_TCP_OUT_PENDING; - } - sws.sock = s; - sws.uptr = uptr; - memset(&(sws.saddr),0,sizeof(struct sockaddr_storage)); - memcpy(&(sws.saddr),remoteAddress,(remoteAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); - - if ((callConnectHandler)&&(connected)) { - try { - _handler->phyOnTcpConnect((PhySocket *)&sws,&(sws.uptr),true); - } catch ( ... ) {} - } - - return (PhySocket *)&sws; - } - - /** - * Attempt to send data to a stream socket (non-blocking) - * - * If -1 is returned, the socket should no longer be used as it is now - * destroyed. If callCloseHandler is true, the close handler will be - * called before the function returns. - * - * This can be used with TCP, Unix, or socket pair sockets. - * - * @param sock An open stream socket (other socket types will fail) - * @param data Data to send - * @param len Length of data - * @param callCloseHandler If true, call close handler on socket closing failure condition (default: true) - * @return Number of bytes actually sent or -1 on fatal error (socket closure) - */ - inline long streamSend(PhySocket *sock,const void *data,unsigned long len,bool callCloseHandler = true) - { - PhySocketImpl &sws = *(reinterpret_cast(sock)); -#if defined(_WIN32) || defined(_WIN64) - long n = (long)::send(sws.sock,reinterpret_cast(data),len,0); - if (n == SOCKET_ERROR) { - switch(WSAGetLastError()) { - case WSAEINTR: - case WSAEWOULDBLOCK: - return 0; - default: - this->close(sock,callCloseHandler); - return -1; - } - } -#else // not Windows - long n = (long)::send(sws.sock,data,len,0); - if (n < 0) { - switch(errno) { -#ifdef EAGAIN - case EAGAIN: -#endif -#if defined(EWOULDBLOCK) && ( !defined(EAGAIN) || (EWOULDBLOCK != EAGAIN) ) - case EWOULDBLOCK: -#endif -#ifdef EINTR - case EINTR: -#endif - return 0; - default: - this->close(sock,callCloseHandler); - return -1; - } - } -#endif // Windows or not - return n; - } - - /** - * For streams, sets whether we want to be notified that the socket is writable - * - * This can be used with TCP, Unix, or socket pair sockets. - * - * Call whack() if this is being done from another thread and you want - * it to take effect immediately. Otherwise it is only guaranteed to - * take effect on the next poll(). - * - * @param sock Stream connection socket - * @param notifyWritable Want writable notifications? - */ - inline const void setNotifyWritable(PhySocket *sock,bool notifyWritable) - { - PhySocketImpl &sws = *(reinterpret_cast(sock)); - if (notifyWritable) { - FD_SET(sws.sock,&_writefds); - } else { - FD_CLR(sws.sock,&_writefds); - } - } - - /** - * Set whether we want to be notified that a socket is readable - * - * This is primarily for raw sockets added with wrapSocket(). It could be - * used with others, but doing so would essentially lock them and prevent - * data from being read from them until this is set to 'true' again. - * - * @param sock Socket to modify - * @param notifyReadable True if socket should be monitored for readability - */ - inline const void setNotifyReadable(PhySocket *sock,bool notifyReadable) - { - PhySocketImpl &sws = *(reinterpret_cast(sock)); - if (notifyReadable) { - FD_SET(sws.sock,&_readfds); - } else { - FD_CLR(sws.sock,&_readfds); - } - } - - /** - * Wait for activity and handle one or more events - * - * Note that this is not guaranteed to wait up to 'timeout' even - * if nothing happens, as whack() or other events such as signals - * may cause premature termination. - * - * @param timeout Timeout in milliseconds or 0 for none (forever) - */ - inline void poll(unsigned long timeout) - { - char buf[131072]; - struct sockaddr_storage ss; - struct timeval tv; - fd_set rfds,wfds,efds; - - memcpy(&rfds,&_readfds,sizeof(rfds)); - memcpy(&wfds,&_writefds,sizeof(wfds)); -#if defined(_WIN32) || defined(_WIN64) - memcpy(&efds,&_exceptfds,sizeof(efds)); -#else - FD_ZERO(&efds); -#endif - - tv.tv_sec = (long)(timeout / 1000); - tv.tv_usec = (long)((timeout % 1000) * 1000); - if (::select((int)_nfds + 1,&rfds,&wfds,&efds,(timeout > 0) ? &tv : (struct timeval *)0) <= 0) - return; - - if (FD_ISSET(_whackReceiveSocket,&rfds)) { - char tmp[16]; -#if defined(_WIN32) || defined(_WIN64) - ::recv(_whackReceiveSocket,tmp,16,0); -#else - ::read(_whackReceiveSocket,tmp,16); -#endif - } - - for(typename std::list::iterator s(_socks.begin());s!=_socks.end();) { - switch (s->type) { - - case ZT_PHY_SOCKET_TCP_OUT_PENDING: -#if defined(_WIN32) || defined(_WIN64) - if (FD_ISSET(s->sock,&efds)) { - this->close((PhySocket *)&(*s),true); - } else // ... if -#endif - if (FD_ISSET(s->sock,&wfds)) { - socklen_t slen = sizeof(ss); - if (::getpeername(s->sock,(struct sockaddr *)&ss,&slen) != 0) { - this->close((PhySocket *)&(*s),true); - } else { - s->type = ZT_PHY_SOCKET_TCP_OUT_CONNECTED; - FD_SET(s->sock,&_readfds); - FD_CLR(s->sock,&_writefds); -#if defined(_WIN32) || defined(_WIN64) - FD_CLR(s->sock,&_exceptfds); -#endif - try { - _handler->phyOnTcpConnect((PhySocket *)&(*s),&(s->uptr),true); - } catch ( ... ) {} - } - } - break; - - case ZT_PHY_SOCKET_TCP_OUT_CONNECTED: - case ZT_PHY_SOCKET_TCP_IN: { - ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable - if (FD_ISSET(sock,&rfds)) { - long n = (long)::recv(sock,buf,sizeof(buf),0); - if (n <= 0) { - this->close((PhySocket *)&(*s),true); - } else { - try { - _handler->phyOnTcpData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n); - } catch ( ... ) {} - } - } - if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { - try { - _handler->phyOnTcpWritable((PhySocket *)&(*s),&(s->uptr)); - } catch ( ... ) {} - } - } break; - - case ZT_PHY_SOCKET_TCP_LISTEN: - if (FD_ISSET(s->sock,&rfds)) { - memset(&ss,0,sizeof(ss)); - socklen_t slen = sizeof(ss); - ZT_PHY_SOCKFD_TYPE newSock = ::accept(s->sock,(struct sockaddr *)&ss,&slen); - if (ZT_PHY_SOCKFD_VALID(newSock)) { - if (_socks.size() >= ZT_PHY_MAX_SOCKETS) { - ZT_PHY_CLOSE_SOCKET(newSock); - } else { -#if defined(_WIN32) || defined(_WIN64) - { BOOL f = (_noDelay ? TRUE : FALSE); setsockopt(newSock,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); } - { u_long iMode=1; ioctlsocket(newSock,FIONBIO,&iMode); } -#else - { int f = (_noDelay ? 1 : 0); setsockopt(newSock,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); } - fcntl(newSock,F_SETFL,O_NONBLOCK); -#endif - _socks.push_back(PhySocketImpl()); - PhySocketImpl &sws = _socks.back(); - FD_SET(newSock,&_readfds); - if ((long)newSock > _nfds) - _nfds = (long)newSock; - sws.type = ZT_PHY_SOCKET_TCP_IN; - sws.sock = newSock; - sws.uptr = (void *)0; - memcpy(&(sws.saddr),&ss,sizeof(struct sockaddr_storage)); - try { - _handler->phyOnTcpAccept((PhySocket *)&(*s),(PhySocket *)&(_socks.back()),&(s->uptr),&(sws.uptr),(const struct sockaddr *)&(sws.saddr)); - } catch ( ... ) {} - } - } - } - break; - - case ZT_PHY_SOCKET_UDP: - if (FD_ISSET(s->sock,&rfds)) { - for(;;) { - memset(&ss,0,sizeof(ss)); - socklen_t slen = sizeof(ss); - long n = (long)::recvfrom(s->sock,buf,sizeof(buf),0,(struct sockaddr *)&ss,&slen); - if (n > 0) { - try { - _handler->phyOnDatagram((PhySocket *)&(*s),&(s->uptr),(const struct sockaddr *)&(s->saddr),(const struct sockaddr *)&ss,(void *)buf,(unsigned long)n); - } catch ( ... ) {} - } else if (n < 0) - break; - } - } - break; - - case ZT_PHY_SOCKET_UNIX_IN: { -#ifdef __UNIX_LIKE__ - ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable - if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { - try { - _handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr),false); - } catch ( ... ) {} - } - if (FD_ISSET(sock,&rfds)) { - long n = (long)::read(sock,buf,sizeof(buf)); - if (n <= 0) { - this->close((PhySocket *)&(*s),true); - } else { - try { - _handler->phyOnUnixData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n); - } catch ( ... ) {} - } - } -#endif // __UNIX_LIKE__ - } break; - - case ZT_PHY_SOCKET_UNIX_LISTEN: -#ifdef __UNIX_LIKE__ - if (FD_ISSET(s->sock,&rfds)) { - memset(&ss,0,sizeof(ss)); - socklen_t slen = sizeof(ss); - ZT_PHY_SOCKFD_TYPE newSock = ::accept(s->sock,(struct sockaddr *)&ss,&slen); - if (ZT_PHY_SOCKFD_VALID(newSock)) { - if (_socks.size() >= ZT_PHY_MAX_SOCKETS) { - ZT_PHY_CLOSE_SOCKET(newSock); - } else { - fcntl(newSock,F_SETFL,O_NONBLOCK); - _socks.push_back(PhySocketImpl()); - PhySocketImpl &sws = _socks.back(); - FD_SET(newSock,&_readfds); - if ((long)newSock > _nfds) - _nfds = (long)newSock; - sws.type = ZT_PHY_SOCKET_UNIX_IN; - sws.sock = newSock; - sws.uptr = (void *)0; - memcpy(&(sws.saddr),&ss,sizeof(struct sockaddr_storage)); - try { - //_handler->phyOnUnixAccept((PhySocket *)&(*s),(PhySocket *)&(_socks.back()),&(s->uptr),&(sws.uptr)); - } catch ( ... ) {} - } - } - } -#endif // __UNIX_LIKE__ - break; - - case ZT_PHY_SOCKET_FD: { - ZT_PHY_SOCKFD_TYPE sock = s->sock; - const bool readable = ((FD_ISSET(sock,&rfds))&&(FD_ISSET(sock,&_readfds))); - const bool writable = ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))); - if ((readable)||(writable)) { - try { - //_handler->phyOnFileDescriptorActivity((PhySocket *)&(*s),&(s->uptr),readable,writable); - } catch ( ... ) {} - } - } break; - - default: - break; - - } - - if (s->type == ZT_PHY_SOCKET_CLOSED) - _socks.erase(s++); - else ++s; - } - } - - /** - * @param sock Socket to close - * @param callHandlers If true, call handlers for TCP connect (success: false) or close (default: true) - */ - inline void close(PhySocket *sock,bool callHandlers = true) - { - if (!sock) - return; - PhySocketImpl &sws = *(reinterpret_cast(sock)); - if (sws.type == ZT_PHY_SOCKET_CLOSED) - return; - - FD_CLR(sws.sock,&_readfds); - FD_CLR(sws.sock,&_writefds); -#if defined(_WIN32) || defined(_WIN64) - FD_CLR(sws.sock,&_exceptfds); -#endif - - if (sws.type != ZT_PHY_SOCKET_FD) - ZT_PHY_CLOSE_SOCKET(sws.sock); - -#ifdef __UNIX_LIKE__ - if (sws.type == ZT_PHY_SOCKET_UNIX_LISTEN) - ::unlink(((struct sockaddr_un *)(&(sws.saddr)))->sun_path); -#endif // __UNIX_LIKE__ - - if (callHandlers) { - switch(sws.type) { - case ZT_PHY_SOCKET_TCP_OUT_PENDING: - try { - _handler->phyOnTcpConnect(sock,&(sws.uptr),false); - } catch ( ... ) {} - break; - case ZT_PHY_SOCKET_TCP_OUT_CONNECTED: - case ZT_PHY_SOCKET_TCP_IN: - try { - _handler->phyOnTcpClose(sock,&(sws.uptr)); - } catch ( ... ) {} - break; - case ZT_PHY_SOCKET_UNIX_IN: -#ifdef __UNIX_LIKE__ - try { - _handler->phyOnUnixClose(sock,&(sws.uptr)); - } catch ( ... ) {} -#endif // __UNIX_LIKE__ - break; - default: - break; - } - } - - // Causes entry to be deleted from list in poll(), ignored elsewhere - sws.type = ZT_PHY_SOCKET_CLOSED; - - if ((long)sws.sock >= (long)_nfds) { - long nfds = (long)_whackSendSocket; - if ((long)_whackReceiveSocket > nfds) - nfds = (long)_whackReceiveSocket; - for(typename std::list::iterator s(_socks.begin());s!=_socks.end();++s) { - if ((s->type != ZT_PHY_SOCKET_CLOSED)&&((long)s->sock > nfds)) - nfds = (long)s->sock; - } - _nfds = nfds; - } - } -}; - -static inline uint64_t _swap64(const uint64_t n) -{ - return ( - ((n & 0x00000000000000FFULL) << 56) | - ((n & 0x000000000000FF00ULL) << 40) | - ((n & 0x0000000000FF0000ULL) << 24) | - ((n & 0x00000000FF000000ULL) << 8) | - ((n & 0x000000FF00000000ULL) >> 8) | - ((n & 0x0000FF0000000000ULL) >> 24) | - ((n & 0x00FF000000000000ULL) >> 40) | - ((n & 0xFF00000000000000ULL) >> 56) - ); -} - -} // namespace ztVsdmInternal - -/*********************************************************************************************************/ - -/** - * No-op update watcher - */ -class vsdm_watcher_noop -{ -public: - template - inline void add(uint64_t,const K &k,const V &v,uint64_t) {} - template - inline void update(uint64_t,const K &k,const V &v,uint64_t) {} - template - inline void del(uint64_t,const K &k) {} -}; - -/** - * No-op cryptor that adds no overhead and does no encryption - */ -class vsdm_cryptor_noop -{ -public: - static inline unsigned long overhead() { return 0; } - inline void encrypt(void *d,unsigned long l) {} - inline bool decrypt(void *d,unsigned long l) { return true; } -}; - -/** - * Default serializer supporting std::string and stdint.h types - */ -class vsdm_default_serializer -{ -public: - static inline unsigned long objectSize(const std::string &o) { return o.length(); } - static inline unsigned long objectSize(const uint8_t o) { return 1; } - static inline unsigned long objectSize(const int8_t o) { return 1; } - static inline unsigned long objectSize(const uint16_t o) { return 2; } - static inline unsigned long objectSize(const int16_t o) { return 2; } - static inline unsigned long objectSize(const uint32_t o) { return 4; } - static inline unsigned long objectSize(const int32_t o) { return 4; } - static inline unsigned long objectSize(const uint64_t o) { return 8; } - static inline unsigned long objectSize(const int64_t o) { return 8; } - - static inline const char *objectData(const std::string &o) { return o.data(); } - static inline const char *objectData(const uint8_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const int8_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const uint16_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const int16_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const uint32_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const int32_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const uint64_t &o) { return reinterpret_cast(&o); } - static inline const char *objectData(const int64_t &o) { return reinterpret_cast(&o); } - - static inline bool objectDeserialize(const char *d,unsigned long l,std::string &o) { o.assign(d,l); return true; } - static inline bool objectDeserialize(const char *d,unsigned long l,uint8_t &o) { if (l == 1) { memcpy(&o,d,1); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,int8_t &o) { if (l == 1) { memcpy(&o,d,1); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,uint16_t &o) { if (l == 2) { memcpy(&o,d,2); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,int16_t &o) { if (l == 2) { memcpy(&o,d,2); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,uint32_t &o) { if (l == 4) { memcpy(&o,d,4); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,int32_t &o) { if (l == 4) { memcpy(&o,d,4); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,uint64_t &o) { if (l == 8) { memcpy(&o,d,8); return true; } else { return false; } } - static inline bool objectDeserialize(const char *d,unsigned long l,int64_t &o) { if (l == 8) { memcpy(&o,d,8); return true; } else { return false; } } -}; - -/** - * VSDM: Very Simple Distributed Map - * - * See README.md for full docs. - * - * @tparam K Key type (must be supported by serializer) - * @tparam V Value type (must be supported by serializer) - * @tparam L Maximum message length (max allowed: UINT32_MAX - 1, default: 131072) - * @tparam W Watcher function (default: vsdm_watcher_noop) - * @tparam S Serializer class with static methods to serialize keys and values (default: vsdm_default_serializer) - * @tparam C Cryptor to encrypt/decrypt and authenticate network traffic (default: vsdm_cryptor_noop) - * @tparam M Map type for underlying data store (default: std::unordered_map) - */ -template< - typename K, - typename V, - unsigned long L = 131072, - typename W = vsdm_watcher_noop, - typename S = vsdm_default_serializer, - typename C = vsdm_cryptor_noop, - template class M = std::unordered_map -> -class vsdm -{ - friend void vsdm_thread_main(void *parent); - friend class ztVsdmInternal::Phy; - -private: - struct vsdm_entry - { - vsdm_entry() : rev(0),deletedAt(0),v() {} - uint64_t rev; - uint64_t deletedAt; - V v; - }; - - struct _connection - { - _connection() : outbuf(),inbuf(),gotHello(false),node(0),sock((ztVsdmInternal::PhySocket *)0) {} - std::string outbuf; - std::string inbuf; - bool gotHello; - uint64_t node; - ztVsdmInternal::PhySocket *sock; - }; - -public: - typedef K key_type; - typedef V value_type; - - /** - * @param id Cluster ID, must be the same on all nodes - * @param node Arbitrary unique node ID - * @param restrictInbound If true, restrict inbound connections to known peer IPs (added via link()) - * @param cryptor Encryptor/decryptor instance (default: C()) - * @param watcher Watcher function instance (default: W()) - */ - vsdm(uint64_t id,uint64_t node,bool restrictInbound,const C &cryptor = C(),const W &watcher = W()) : - _node(node), - _id(id), - _rev(0), - _connections(), - _m(), - _lock(), - _phy(this,false,false), - _cryptor(cryptor), - _watcher(watcher), - _run(true), - _restrictInbound(restrictInbound), - _t(_threadMain,reinterpret_cast(this)) - { - } - - ~vsdm() - { - _run = false; - _phy.whack(); - _t.join(); - } - - /** - * @param k Key to set - * @param v New value for key - * @return Revision of entry in map - */ - inline uint64_t set(const K &k,const V &v) - { - std::lock_guard l(_lock); - - vsdm_entry &e = _m[k]; - e.rev = ++_rev; - e.deletedAt = 0; - e.v = v; - - std::vector sentToNodes; - for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { - if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { - sendUpdate(c2->second,k,e); - sentToNodes.push_back(c2->second.node); -#ifdef VSDM_DEBUG - fprintf(stderr,">> %lu: %s=%s\n",(unsigned long)c2->second.node,k.c_str(),v.c_str()); fflush(stderr); -#endif - } - } - _phy.whack(); - - return _rev; - } - - /** - * @param k Key to check - * @return Revision of key that we have or 0 if not found - */ - inline uint64_t have(const K &k) const - { - std::lock_guard l(_lock); - typename std::unordered_map::const_iterator i(_m.find(k)); - if ((i == _m.end())||(i->second.deletedAt)) - return 0; - return i->second.rev; - } - - /** - * @param k Key to get - * @param dfl Default value if key is not found - * @param have If non-NULL, set to revision of this key or 0 if not found - * @return Key value or dfl if not found - */ - inline V get(const K &k,const V &dfl = V(),uint64_t *have = (uint64_t *)0) const - { - std::lock_guard l(_lock); - typename std::unordered_map::const_iterator i(_m.find(k)); - if ((i == _m.end())||(i->second.deletedAt)) { - if (have) - *have = 0; - return dfl; - } - if (have) - *have = i->second.rev; - return i->second.v; - } - - /** - * @param k Key to get - * @param have If non-NULL, set to revision of this key or 0 if not found - * @return Key's value or default/empty V() if not found - */ - inline V get(const K &k,uint64_t *have) const - { - return get(k,V(),have); - } - - /** - * Erase a key - * - * Erased entries are not wholly purged from memory immediately. They - * are marked as erased and purged after sufficient time for propagation. - * - * @param k Key to erase - * @return Previous revision of this key in map or 0 if not found - */ - inline bool del(const K &k) - { - uint64_t prev = 0; - std::lock_guard l(_lock); - - typename std::unordered_map::iterator i(_m.find(k)); - if (i == _m.end()) - return 0; - prev = i->second.rev; - i->second.rev = ++_rev; - i->second.deletedAt = _rev; - i->second.v.clear(); - - std::vector sentToNodes; - for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { - if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { - sendUpdate(c2->second,k,i->second); - sentToNodes.push_back(c2->second.node); -#ifdef VSDM_DEBUG - fprintf(stderr,">> %lu: %s=\n",(unsigned long)c2->second.node,k.c_str()); fflush(stderr); -#endif - } - } - _phy.whack(); - - return prev; - } - - /** - * Listen for incoming node connections on an address - * - * This can be called more than once to listen on more than one address and port. - * - * @param sa Socket address - * @return True if bind succeeded - */ - inline bool listen(const struct sockaddr *sa) - { - std::lock_guard l(_lock); - return (_phy.tcpListen(sa) != (ztVsdmInternal::PhySocket *)0); - } - - /** - * Add a remote node endpoint - * - * This can be called for an arbitrary number of other endpoints in the - * network to tell this node to attempt to maintain a link to them. - * - * @param node Node ID of remote - * @param sa Socket address of remote - * @param salen Length of socket address structure - */ - inline void link(uint64_t node,const struct sockaddr_in *sa,unsigned int salen) - { - std::lock_guard l(_lock); - if ((node != _node)&&(salen <= sizeof(struct sockaddr_storage))) - memcpy(&(_peers[node]),sa,salen); - } - - /** - * @return Node IDs of nodes that are currently connected - */ - inline std::vector who() const - { - std::vector w; - std::lock_guard l(_lock); - for(typename std::unordered_map::const_iterator i(_connections.begin());i!=_connections.end();++i) { - if ((i->gotHello)&&(std::find(w.begin(),w.end(),i->second.node) == w.end())) - w.push_back(i->second.node); - } - return w; - } - - /** - * @return True if we are currently connected to at least one other node - */ - inline bool connected() const - { - std::lock_guard l(_lock); - for(typename std::unordered_map::const_iterator i(_connections.begin());i!=_connections.end();++i) { - if (i->gotHello) - return true; - } - return false; - } - - /** - * Iterate through all members of this map, with optional deletion - * - * The function is executed against all key/value pairs and returns a signed integer. - * A negative return value causes the entry to be deleted, while a positive return - * value means the key's value (which is passed into the function as a reference) has - * been modified and should be replicated. A return value of zero means no change. - * - * Other methods should not be called since doing so can result in a deadlock. - * - * @param func Function to execute against all members of map, returns integer (see description) - */ - template - inline void each(F func) - { - std::vector sentToNodes; - bool whack = false; - std::lock_guard l(_lock); - for(typename M::iterator i(_m.begin());i!=_m.end();++i) { - if (!i->second.deletedAt) { - try { - const int result = func(i->first,i->second.v); - if (result < 0) { - i->second.rev = ++_rev; - i->second.deletedAt = _rev; - i->second.v.clear(); - } else if (result > 0) { - i->second.rev = ++_rev; - i->second.deletedAt = 0; - // v will have been modified in place - } - if (result != 0) { - sentToNodes.clear(); - for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { - if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { - sendUpdate(c2->second,i->first,i->second); - sentToNodes.push_back(c2->second.node); - } - } - whack = true; - } - } catch ( ... ) {} - } - } - if (whack) - _phy.whack(); - } - -private: - inline vsdm &operator=(const vsdm &v) { return *this; } - - static void _threadMain(void *p) { reinterpret_cast(p)->threadMain(); } - inline void threadMain() - { - std::vector haveNodes; - time_t lastcheck = 0; - time_t lastclean = 0; - while (_run) { - _phy.poll(1000); - - time_t now = time(0); - - // Check connections with other nodes and try to establish them - // if they're not present. - if ((now - lastcheck) >= 2) { - lastcheck = now; - haveNodes.clear(); - - std::lock_guard l(_lock); - - for(typename std::unordered_map::const_iterator c(_connections.begin());c!=_connections.end();++c) { - if (std::find(haveNodes.begin(),haveNodes.end(),c->second.node) == haveNodes.end()) - haveNodes.push_back(c->second.node); - } - - for(std::unordered_map::iterator p(_peers.begin());p!=_peers.end();++p) { - if (std::find(haveNodes.begin(),haveNodes.end(),p->first) == haveNodes.end()) { - bool connected = false; - ztVsdmInternal::PhySocket *ns = _phy.tcpConnect((const struct sockaddr *)&(p->second),connected,(void *)0,true); - if (ns) { - _connection &c = _connections[ns]; - c.gotHello = false; - c.node = p->first; - c.sock = ns; - } - } - } - } - - // Forget deleted entries if they've had ample time to propagate - if ((now - lastclean) >= 120) { - lastclean = now; - uint64_t delHorizon = _m.size() * (2 + _peers.size()); - if (_rev > delHorizon) { - delHorizon -= _rev; - std::lock_guard l(_lock); - for(typename M::iterator i(_m.begin());i!=_m.end();) { - if ((i->second.deletedAt > 0)&&(i->second.deletedAt < delHorizon)) - _m.erase(i++); - else ++i; - } - } - } - } - } - - inline void sendUpdate(_connection &c,const std::string &k,const vsdm_entry &e) - { - // assumes lock is locked - const uint32_t ks = (uint32_t)S::objectSize(k); - uint32_t vs = 0; - uint32_t hdr[4]; - hdr[0] = htonl((uint32_t)((e.rev >> 32) & 0xffffffff)); - hdr[1] = htonl((uint32_t)(e.rev & 0xffffffff)); - hdr[2] = htonl(ks); - if (e.deletedAt) { - hdr[3] = 0xffffffff; - } else { - vs = (uint32_t)S::objectSize(e.v); - hdr[3] = htonl(vs); - } - - const uint32_t s = htonl((uint32_t)(16 + C::overhead() + ks + vs)); - c.outbuf.append((const char *)&s,4); - - const unsigned long start = (unsigned long)c.outbuf.length(); - c.outbuf.append((const char *)hdr,16); - c.outbuf.append(S::objectData(k),ks); - if (!e.deletedAt) - c.outbuf.append(S::objectData(e.v),vs); - c.outbuf.append(C::overhead(),(char)0); - const unsigned long end = (unsigned long)c.outbuf.length(); - - _cryptor.encrypt(reinterpret_cast(const_cast(c.outbuf.data()) + start),end - start); - - _phy.setNotifyWritable(c.sock,true); - } - - inline void sendUpdateToAll(ztVsdmInternal::PhySocket *receivedOnSock,const uint64_t receivedFromNode,const std::string &k,const vsdm_entry &e) - { - // assumes lock is locked - std::vector sentToNodes; - for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { - if ((c2->first != receivedOnSock)&&(c2->second.gotHello)&&(c2->second.node != receivedFromNode)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { - sendUpdate(c2->second,k,e); - sentToNodes.push_back(c2->second.node); -#ifdef VSDM_DEBUG - fprintf(stderr,">> %lu: %s=%s\n",(unsigned long)c2->second.node,k.c_str(),(e.deletedAt) ? "" : e.v.c_str()); fflush(stderr); -#endif - } - } - _phy.whack(); - } - - inline void sendHello(ztVsdmInternal::PhySocket *sock) - { - uint64_t hdr[3]; - if (htonl(1) == 1) { - hdr[0] = _node; - hdr[1] = _id; - hdr[2] = _rev; - } else { - hdr[0] = ztVsdmInternal::_swap64(_node); - hdr[1] = ztVsdmInternal::_swap64(_id); - hdr[2] = ztVsdmInternal::_swap64(_rev); - } - uint8_t tmp[24 + C::overhead()]; - memcpy(tmp,hdr,24); - _cryptor.encrypt(reinterpret_cast(tmp),sizeof(tmp)); - _phy.streamSend(sock,reinterpret_cast(tmp),sizeof(tmp)); - } - - inline void phyOnTcpConnect(ztVsdmInternal::PhySocket *sock,void **uptr,bool success) - { - std::lock_guard l(_lock); - if (success) { - _connection &c = _connections[sock]; - c.gotHello = false; - c.sock = sock; - *uptr = (void *)&c; - sendHello(sock); - } else { - _connections.erase(sock); - } - } - - inline void phyOnTcpAccept(ztVsdmInternal::PhySocket *sockL,ztVsdmInternal::PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from) - { - std::lock_guard l(_lock); - - if (_restrictInbound) { - bool ok = false; - for(typename std::unordered_map::const_iterator i(_peers.begin());i!=_peers.end();++i) { - if (from->sa_family == i->second.ss_family) { - if ( (from->sa_family == AF_INET) && (reinterpret_cast(from)->sin_addr.s_addr == reinterpret_cast(&(i->second))->sin_addr.s_addr) ) { - ok = true; - break; - } else if ( (from->sa_family == AF_INET6) && (memcmp(reinterpret_cast(from)->sin6_addr.s6_addr,reinterpret_cast(&(i->second))->sin6_addr.s6_addr,16) == 0) ) { - ok = true; - break; - } - } - } - if (!ok) { -#ifdef VSDM_DEBUG - fprintf(stderr," * dropped inbound connection: peer not from a known IP address\n"); fflush(stderr); -#endif - _phy.close(sockN,false); - return; - } - } - - _connection &c = _connections[sockN]; - c.gotHello = false; - c.node = _node; // impossible value for a remote - c.sock = sockN; - *uptrN = (void *)&c; - sendHello(sockN); - } - - inline void phyOnTcpClose(ztVsdmInternal::PhySocket *sock,void **uptr) - { - std::lock_guard l(_lock); - _connections.erase(sock); - } - - inline void phyOnTcpData(ztVsdmInternal::PhySocket *sock,void **uptr,void *data,unsigned long len) - { - _connection *const c = (_connection *)*uptr; - if (!c) return; - - std::unique_lock l(_lock); - c->inbuf.append(reinterpret_cast(data),len); - for(;;) { - if (c->gotHello) { - - if (c->inbuf.length() >= 20) { // got message size and header - uint32_t _totalLen; - memcpy(&_totalLen,c->inbuf.data(),4); - const unsigned long totalLen = ntohl(_totalLen); - if ((totalLen > L)||(totalLen < 16)) { // message too small or too large - _connections.erase(sock); - _phy.close(sock,false); - return; - } - - if (c->inbuf.length() >= (4 + totalLen)) { // got full message - - if (!_cryptor.decrypt(reinterpret_cast(const_cast(c->inbuf.data()) + 4),totalLen)) { - _connections.erase(sock); - _phy.close(sock,false); - return; - } - - uint32_t hdr[4]; - memcpy(hdr,c->inbuf.data() + 4,16); - - const uint64_t objectRev = ((uint64_t)ntohl(hdr[0]) << 32) | (uint64_t)ntohl(hdr[1]); - const unsigned long keyLen = (unsigned long)ntohl(hdr[2]); - unsigned long valueLen = (unsigned long)ntohl(hdr[3]); - - if (objectRev > _rev) - _rev = objectRev; - - uint64_t deletedAt = 0; - if (valueLen == 0xffffffff) { - valueLen = 0; - deletedAt = _rev; - } - if ((keyLen + valueLen + 16 + C::overhead()) > totalLen) { // key and/or value length invalid - _connections.erase(sock); - _phy.close(sock,false); - return; - } - - K k; - if (!S::objectDeserialize(c->inbuf.data() + 16 + 4,keyLen,k)) { - _connections.erase(sock); - _phy.close(sock,false); - return; - } - - vsdm_entry &e = _m[k]; - if (e.rev < objectRev) { - const bool added = (e.rev == 0); - e.rev = objectRev; - e.deletedAt = deletedAt; - if (e.deletedAt) { - e.v = V(); - } else { - if (!S::objectDeserialize(c->inbuf.data() + 16 + 4 + keyLen,valueLen,e.v)) { - _connections.erase(sock); - _phy.close(sock,false); - return; - } - } -#ifdef VSDM_DEBUG - fprintf(stderr,"<< %lu: %s=%s\n",(unsigned long)c->node,k.c_str(),(deletedAt) ? "" : e.v.c_str()); fflush(stderr); -#endif - sendUpdateToAll(sock,c->node,k,e); - - l.unlock(); - try { - if (added) { - _watcher.add(c->node,k,e.v,objectRev); - } else if (deletedAt) { - _watcher.del(c->node,k); - } else { - _watcher.update(c->node,k,e.v,objectRev); - } - } catch ( ... ) {} - l.lock(); - } - - c->inbuf.erase(c->inbuf.begin(),c->inbuf.begin() + totalLen + 4); - - // continue and process more messages in queue, if any - } else { // still waiting on full message - break; - } - } else { // still waiting on message size and header - break; - } - - } else if (c->inbuf.length() >= (24 + C::overhead())) { // got hello header - - if (!_cryptor.decrypt(reinterpret_cast(const_cast(c->inbuf.data())),24 + C::overhead())) { - _connections.erase(sock); - _phy.close(sock,false); - return; - } - - uint64_t hdr[3]; - memcpy(hdr,c->inbuf.data(),24); - c->inbuf.erase(c->inbuf.begin(),c->inbuf.begin() + 24 + C::overhead()); - - if (htonl(1) != 1) { - hdr[0] = ztVsdmInternal::_swap64(hdr[0]); - hdr[1] = ztVsdmInternal::_swap64(hdr[1]); - hdr[2] = ztVsdmInternal::_swap64(hdr[2]); - } - - if ((hdr[0] == _node)||(hdr[1] != _id)) { // don't connect to self, and don't connect to other map IDs - _connections.erase(sock); - _phy.close(sock,false); - break; - } else { - c->gotHello = true; - c->node = hdr[0]; - - if (hdr[2] > _rev) - _rev = hdr[2]; - - for(typename M::const_iterator i(_m.begin());i!=_m.end();++i) { - if (i->second.rev >= hdr[2]) { - sendUpdate(*c,i->first,i->second); -#ifdef VSDM_DEBUG - fprintf(stderr,">> %lu: %s=%s (new link)\n",(unsigned long)c->node,i->first.c_str(),i->second.v.c_str()); fflush(stderr); -#endif - } - } - _phy.whack(); - } - - // continue and process more messages in queue, if any - } else { // still waiting on hello header - break; - } - } - } - - inline void phyOnTcpWritable(ztVsdmInternal::PhySocket *sock,void **uptr) - { - std::lock_guard l(_lock); - _connection *c = (_connection *)*uptr; - if (c) { - if (c->outbuf.length() > 0) { - long n = _phy.streamSend(sock,c->outbuf.data(),c->outbuf.length()); - if (n <= 0) { - _connections.erase(sock); - _phy.close(sock,false); - return; - } else if (n == (long)c->outbuf.length()) { - c->outbuf.clear(); - } else { - c->outbuf.erase(c->outbuf.begin(),c->outbuf.begin() + n); - } - } - if (c->outbuf.length() == 0) { - _phy.setNotifyWritable(c->sock,false); - } - } - } - - inline void phyOnDatagram(ztVsdmInternal::PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) {} - inline void phyOnFileDescriptorActivity(ztVsdmInternal::PhySocket *sock,void **uptr,bool readable,bool writable) {} - inline void phyOnUnixAccept(ztVsdmInternal::PhySocket *sockL,ztVsdmInternal::PhySocket *sockN,void **uptrL,void **uptrN) {} - inline void phyOnUnixClose(ztVsdmInternal::PhySocket *sock,void **uptr) {} - inline void phyOnUnixData(ztVsdmInternal::PhySocket *sock,void **uptr,void *data,unsigned long len) {} - inline void phyOnUnixWritable(ztVsdmInternal::PhySocket *sock,void **uptr) {} - - const uint64_t _node; - const uint64_t _id; - uint64_t _rev; - std::unordered_map _peers; - std::unordered_map _connections; - M _m; - mutable std::mutex _lock; - ztVsdmInternal::Phy _phy; - C _cryptor; - W _watcher; - volatile bool _run; - bool _restrictInbound; - std::thread _t; -}; - -#endif diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 4709b116..74fa4301 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1089,6 +1089,62 @@ typedef struct unsigned long peerCount; } ZT_PeerList; +/** + * Types of stored objects that the core may wish to save or load + */ +enum ZT_StoredObjectType +{ + /** + * Node status information (reserved, not currently used) + */ + ZT_STORED_OBJECT_STATUS = 0, + + /** + * String serialized public identity + */ + ZT_STORED_OBJECT_IDENTITY_PUBLIC = 1, + + /** + * String serialized secret identity + */ + ZT_STORED_OBJECT_IDENTITY_SECRET = 1, + + /** + * Binary serialized peer state + */ + ZT_STORED_OBJECT_PEER = 3, + + /** + * Identity (other node, not this one) + */ + ZT_STORED_OBJECT_IDENTITY = 4, + + /** + * Network configuration object + */ + ZT_STORED_OBJECT_NETWORK_CONFIG = 5, + + /** + * Planet definition (object ID will be zero and should be ignored since there's only one) + */ + ZT_STORED_OBJECT_PLANET = 6, + + /** + * Moon definition + */ + ZT_STORED_OBJECT_MOON = 7, + + /** + * Multicast membership + */ + ZT_STORED_OBJECT_MULTICAST_MEMBERSHIP = 8, + + /** + * IDs above this are never used by the core and are available for implementation use + */ + ZT_STORED_OBJECT__MAX_TYPE_ID = 255 +}; + /** * An instance of a ZeroTier One node (opaque) */ diff --git a/node/Buffer.hpp b/node/Buffer.hpp index 8e6b78fd..fea32767 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -93,7 +93,6 @@ public: } Buffer(unsigned int l) - throw(std::out_of_range) { if (l > C) throw std::out_of_range("Buffer: construct with size larger than capacity"); @@ -102,51 +101,42 @@ public: template Buffer(const Buffer &b) - throw(std::out_of_range) { *this = b; } Buffer(const void *b,unsigned int l) - throw(std::out_of_range) { copyFrom(b,l); } Buffer(const std::string &s) - throw(std::out_of_range) { copyFrom(s.data(),s.length()); } template inline Buffer &operator=(const Buffer &b) - throw(std::out_of_range) { if (unlikely(b._l > C)) throw std::out_of_range("Buffer: assignment from buffer larger than capacity"); - memcpy(_b,b._b,_l = b._l); - return *this; - } - - inline Buffer &operator=(const std::string &s) - throw(std::out_of_range) - { - copyFrom(s.data(),s.length()); + if (C2 == C) { + memcpy(this,&b,sizeof(Buffer)); + } else { + memcpy(_b,b._b,_l = b._l); + } return *this; } inline void copyFrom(const void *b,unsigned int l) - throw(std::out_of_range) { if (unlikely(l > C)) throw std::out_of_range("Buffer: set from C array larger than capacity"); - _l = l; memcpy(_b,b,l); + _l = l; } unsigned char operator[](const unsigned int i) const - throw(std::out_of_range) { if (unlikely(i >= _l)) throw std::out_of_range("Buffer: [] beyond end of data"); @@ -154,7 +144,6 @@ public: } unsigned char &operator[](const unsigned int i) - throw(std::out_of_range) { if (unlikely(i >= _l)) throw std::out_of_range("Buffer: [] beyond end of data"); @@ -175,14 +164,12 @@ public: * @throws std::out_of_range Field extends beyond data size */ unsigned char *field(unsigned int i,unsigned int l) - throw(std::out_of_range) { if (unlikely((i + l) > _l)) throw std::out_of_range("Buffer: field() beyond end of data"); return (unsigned char *)(_b + i); } const unsigned char *field(unsigned int i,unsigned int l) const - throw(std::out_of_range) { if (unlikely((i + l) > _l)) throw std::out_of_range("Buffer: field() beyond end of data"); @@ -198,7 +185,6 @@ public: */ template inline void setAt(unsigned int i,const T v) - throw(std::out_of_range) { if (unlikely((i + sizeof(T)) > _l)) throw std::out_of_range("Buffer: setAt() beyond end of data"); @@ -221,7 +207,6 @@ public: */ template inline T at(unsigned int i) const - throw(std::out_of_range) { if (unlikely((i + sizeof(T)) > _l)) throw std::out_of_range("Buffer: at() beyond end of data"); @@ -248,7 +233,6 @@ public: */ template inline void append(const T v) - throw(std::out_of_range) { if (unlikely((_l + sizeof(T)) > C)) throw std::out_of_range("Buffer: append beyond capacity"); @@ -271,7 +255,6 @@ public: * @throws std::out_of_range Attempt to append beyond capacity */ inline void append(unsigned char c,unsigned int n) - throw(std::out_of_range) { if (unlikely((_l + n) > C)) throw std::out_of_range("Buffer: append beyond capacity"); @@ -287,7 +270,6 @@ public: * @throws std::out_of_range Attempt to append beyond capacity */ inline void append(const void *b,unsigned int l) - throw(std::out_of_range) { if (unlikely((_l + l) > C)) throw std::out_of_range("Buffer: append beyond capacity"); @@ -302,7 +284,6 @@ public: * @throws std::out_of_range Attempt to append beyond capacity */ inline void append(const std::string &s) - throw(std::out_of_range) { append(s.data(),(unsigned int)s.length()); } @@ -314,7 +295,6 @@ public: * @throws std::out_of_range Attempt to append beyond capacity */ inline void appendCString(const char *s) - throw(std::out_of_range) { for(;;) { if (unlikely(_l >= C)) @@ -333,7 +313,6 @@ public: */ template inline void append(const Buffer &b) - throw(std::out_of_range) { append(b._b,b._l); } @@ -349,7 +328,6 @@ public: * @return Pointer to beginning of appended field of length 'l' */ inline char *appendField(unsigned int l) - throw(std::out_of_range) { if (unlikely((_l + l) > C)) throw std::out_of_range("Buffer: append beyond capacity"); @@ -367,7 +345,6 @@ public: * @throws std::out_of_range Capacity exceeded */ inline void addSize(unsigned int i) - throw(std::out_of_range) { if (unlikely((i + _l) > C)) throw std::out_of_range("Buffer: setSize to larger than capacity"); @@ -383,7 +360,6 @@ public: * @throws std::out_of_range Size larger than capacity */ inline void setSize(const unsigned int i) - throw(std::out_of_range) { if (unlikely(i > C)) throw std::out_of_range("Buffer: setSize to larger than capacity"); @@ -397,7 +373,6 @@ public: * @throw std::out_of_range Position is beyond size of buffer */ inline void behead(const unsigned int at) - throw(std::out_of_range) { if (!at) return; @@ -414,7 +389,6 @@ public: * @throw std::out_of_range Position plus length is beyond size of buffer */ inline void erase(const unsigned int at,const unsigned int length) - throw(std::out_of_range) { const unsigned int endr = at + length; if (unlikely(endr > _l)) @@ -495,8 +469,8 @@ public: } private: - unsigned int _l; char ZT_VAR_MAY_ALIAS _b[C]; + unsigned int _l; }; } // namespace ZeroTier diff --git a/node/Constants.hpp b/node/Constants.hpp index 3974f0ec..fbbba76e 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -150,6 +150,12 @@ #endif #endif +#ifdef __WINDOWS__ +#define ZT_PACKED_STRUCT(D) __pragma(pack(push,1)) D __pragma(pack(pop)) +#else +#define ZT_PACKED_STRUCT(D) D __attribute__((packed)) +#endif + /** * Length of a ZeroTier address in bytes */ diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index c46ed68f..b702f608 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -374,12 +374,7 @@ private: } static inline unsigned long _hc(const uint64_t i) { - /* NOTE: this assumes that 'i' is evenly distributed, which is the case for - * packet IDs and network IDs -- the two use cases in ZT for uint64_t keys. - * These values are also greater than 0xffffffff so they'll map onto a full - * bucket count just fine no matter what happens. Normally you'd want to - * hash an integer key index in a hash table. */ - return (unsigned long)i; + return (unsigned long)(i ^ (i >> 32)); // good for network IDs and addresses } static inline unsigned long _hc(const uint32_t i) { -- cgit v1.2.3 From 76452b4e281b83a93c923a9e144b10797f7d4066 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Jun 2017 07:39:31 -0700 Subject: Data structure fixup. --- node/Node.cpp | 69 ++++++++++++++++++++++++++++++++--------------------------- node/Node.hpp | 32 +++++++++++---------------- 2 files changed, 49 insertions(+), 52 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 8849e0f4..bc4b858e 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -238,10 +238,13 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint std::vector< SharedPtr > needConfig; { Mutex::Lock _l(_networks_m); - for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { - if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig())) - needConfig.push_back(n->second); - n->second->sendUpdatesToMembers(tptr); + Hashtable< uint64_t,SharedPtr >::Iterator i(_networks); + uint64_t *k = (uint64_t *)0; + SharedPtr *v = (SharedPtr *)0; + while (i.next(k,v)) { + if (((now - (*v)->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!(*v)->hasConfig())) + needConfig.push_back(*v); + (*v)->sendUpdatesToMembers(tptr); } } for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) @@ -306,34 +309,30 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr) { Mutex::Lock _l(_networks_m); - SharedPtr nw = _network(nwid); - if(!nw) { - const std::pair< uint64_t,SharedPtr > nn(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr))); - _networks.insert(std::upper_bound(_networks.begin(),_networks.end(),nn),nn); - } + SharedPtr &nw = _networks[nwid]; + if (!nw) + nw = SharedPtr(new Network(RR,tptr,nwid,uptr)); return ZT_RESULT_OK; } ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { ZT_VirtualNetworkConfig ctmp; - std::vector< std::pair< uint64_t,SharedPtr > > newn; void **nUserPtr = (void **)0; - Mutex::Lock _l(_networks_m); - for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { - if (n->first != nwid) { - newn.push_back(*n); - } else { - if (uptr) - *uptr = *n->second->userPtr(); - n->second->externalConfig(&ctmp); - n->second->destroy(); - nUserPtr = n->second->userPtr(); - } + { + Mutex::Lock _l(_networks_m); + SharedPtr *nw = _networks.get(nwid); + if (!nw) + return ZT_RESULT_OK; + if (uptr) + *uptr = (*nw)->userPtr(); + (*nw)->externalConfig(&ctmp); + (*nw)->destroy(); + nUserPtr = (*nw)->userPtr(); + _networks.erase(nwid); } - _networks.swap(newn); - + if (nUserPtr) RR->node->configureVirtualNetworkPort(tptr,nwid,nUserPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); @@ -431,10 +430,10 @@ ZT_PeerList *Node::peers() const ZT_VirtualNetworkConfig *Node::networkConfig(uint64_t nwid) const { Mutex::Lock _l(_networks_m); - SharedPtr nw = _network(nwid); - if(nw) { + const SharedPtr *nw = _networks.get(nwid); + if (nw) { ZT_VirtualNetworkConfig *nc = (ZT_VirtualNetworkConfig *)::malloc(sizeof(ZT_VirtualNetworkConfig)); - nw->externalConfig(nc); + (*nw)->externalConfig(nc); return nc; } return (ZT_VirtualNetworkConfig *)0; @@ -451,8 +450,11 @@ ZT_VirtualNetworkList *Node::networks() const nl->networks = (ZT_VirtualNetworkConfig *)(buf + sizeof(ZT_VirtualNetworkList)); nl->networkCount = 0; - for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) - n->second->externalConfig(&(nl->networks[nl->networkCount++])); + Hashtable< uint64_t,SharedPtr >::Iterator i(*const_cast< Hashtable< uint64_t,SharedPtr > *>(&_networks)); + uint64_t *k = (uint64_t *)0; + SharedPtr *v = (SharedPtr *)0; + while (i.next(k,v)) + (*v)->externalConfig(&(nl->networks[nl->networkCount++])); return nl; } @@ -601,10 +603,13 @@ bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,cons { Mutex::Lock _l(_networks_m); - for(std::vector< std::pair< uint64_t, SharedPtr > >::const_iterator i=_networks.begin();i!=_networks.end();++i) { - if (i->second->hasConfig()) { - for(unsigned int k=0;ksecond->config().staticIpCount;++k) { - if (i->second->config().staticIps[k].containsAddress(remoteAddress)) + Hashtable< uint64_t,SharedPtr >::Iterator i(_networks); + uint64_t *k = (uint64_t *)0; + SharedPtr *v = (SharedPtr *)0; + while (i.next(k,v)) { + if ((*v)->hasConfig()) { + for(unsigned int k=0;k<(*v)->config().staticIpCount;++k) { + if ((*v)->config().staticIps[k].containsAddress(remoteAddress)) return false; } } diff --git a/node/Node.hpp b/node/Node.hpp index 006551fa..92e830c5 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -46,6 +46,7 @@ #include "Path.hpp" #include "Salsa20.hpp" #include "NetworkController.hpp" +#include "Hashtable.hpp" #undef TRACE #ifdef ZT_TRACE @@ -154,26 +155,27 @@ public: inline SharedPtr network(uint64_t nwid) const { Mutex::Lock _l(_networks_m); - return _network(nwid); + const SharedPtr *n = _networks.get(nwid); + if (n) + return *n; + return SharedPtr(); } inline bool belongsToNetwork(uint64_t nwid) const { Mutex::Lock _l(_networks_m); - for(std::vector< std::pair< uint64_t, SharedPtr > >::const_iterator i=_networks.begin();i!=_networks.end();++i) { - if (i->first == nwid) - return true; - } - return false; + return _networks.contains(nwid); } inline std::vector< SharedPtr > allNetworks() const { std::vector< SharedPtr > nw; Mutex::Lock _l(_networks_m); - nw.reserve(_networks.size()); - for(std::vector< std::pair< uint64_t, SharedPtr > >::const_iterator i=_networks.begin();i!=_networks.end();++i) - nw.push_back(i->second); + Hashtable< uint64_t,SharedPtr >::Iterator i(*const_cast< Hashtable< uint64_t,SharedPtr > * >(&_networks)); + uint64_t *k = (uint64_t *)0; + SharedPtr *v = (SharedPtr *)0; + while (i.next(k,v)) + nw.push_back(*v); return nw; } @@ -266,16 +268,6 @@ public: virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); private: - inline SharedPtr _network(uint64_t nwid) const - { - // assumes _networks_m is locked - for(std::vector< std::pair< uint64_t, SharedPtr > >::const_iterator i=_networks.begin();i!=_networks.end();++i) { - if (i->first == nwid) - return i->second; - } - return SharedPtr(); - } - RuntimeEnvironment _RR; RuntimeEnvironment *RR; void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P @@ -288,7 +280,7 @@ private: // Time of last identity verification indexed by InetAddress.rateGateHash() -- used in IncomingPacket::_doHELLO() via rateGateIdentityVerification() uint64_t _lastIdentityVerification[16384]; - std::vector< std::pair< uint64_t, SharedPtr > > _networks; + Hashtable< uint64_t,SharedPtr > _networks; Mutex _networks_m; std::vector _directPaths; -- cgit v1.2.3 From 6015b529a0a8615a0208b96236d70ba501f327ee Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Jun 2017 12:33:05 -0700 Subject: More clustering work. --- include/ZeroTierOne.h | 215 ++++++++++++++++++++++++++++++++------------------ node/Network.cpp | 39 ++------- node/Network.hpp | 3 +- node/Node.cpp | 166 ++++++++++++++++++++++++++++++-------- node/Node.hpp | 15 ++-- node/Topology.cpp | 70 +++++++--------- 6 files changed, 323 insertions(+), 185 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 74fa4301..50c47876 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -271,22 +271,27 @@ enum ZT_ResultCode */ ZT_RESULT_OK = 0, - // Fatal errors (>0, <1000) + /** + * Call produced no error but no action was taken + */ + ZT_RESULT_OK_IGNORED = 1, + + // Fatal errors (>100, <1000) /** * Ran out of memory */ - ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY = 1, + ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY = 100, /** * Data store is not writable or has failed */ - ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED = 2, + ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED = 101, /** * Internal error (e.g. unexpected exception indicating bug or build problem) */ - ZT_RESULT_FATAL_ERROR_INTERNAL = 3, + ZT_RESULT_FATAL_ERROR_INTERNAL = 102, // Non-fatal errors (>1000) @@ -1090,59 +1095,97 @@ typedef struct } ZT_PeerList; /** - * Types of stored objects that the core may wish to save or load + * ZeroTier core state objects + * + * All of these objects can be persisted if desired. To preserve the + * identity of a node and its address, the identity (public and secret) + * must be saved at a minimum. + * + * The reference service implementation currently persists identity, + * peer identities (for a period of time), planet, moons, and network + * configurations. Other state is treated as ephemeral. + * + * All state objects should be replicated in cluster mode. The reference + * clustering implementation uses a rumor mill algorithm in which state + * updates that are accepted with RESULT_OK (but not RESULT_OK_IGNORED) + * are flooded to all connected cluster peers. This results in updates + * being flooded across the cluster until all cluster members have the + * latest. */ -enum ZT_StoredObjectType +enum ZT_StateObjectType { /** - * Node status information (reserved, not currently used) - */ - ZT_STORED_OBJECT_STATUS = 0, - - /** - * String serialized public identity + * Null object -- ignored */ - ZT_STORED_OBJECT_IDENTITY_PUBLIC = 1, + ZT_STATE_OBJECT_NULL = 0, /** - * String serialized secret identity + * identity.public + * + * Object ID: this node's address if known, or 0 if unknown (first query) + * Canonical path: /identity.public + * Persistence: required */ - ZT_STORED_OBJECT_IDENTITY_SECRET = 1, + ZT_STATE_OBJECT_IDENTITY_PUBLIC = 1, /** - * Binary serialized peer state + * identity.secret + * + * Object ID: this node's address if known, or 0 if unknown (first query) + * Canonical path: /identity.public + * Persistence: required, should be stored with restricted permissions e.g. mode 0600 on *nix */ - ZT_STORED_OBJECT_PEER = 3, + ZT_STATE_OBJECT_IDENTITY_SECRET = 2, /** - * Identity (other node, not this one) + * A peer to which this node is communicating + * + * Object ID: peer address + * Canonical path: /peers.d/
(10-digit hex address) + * Persistence: optional, can be purged at any time */ - ZT_STORED_OBJECT_IDENTITY = 4, + ZT_STATE_OBJECT_PEER = 3, /** - * Network configuration object + * The identity of a known peer + * + * Object ID: peer address + * Canonical path: /iddb.d/
(10-digit hex address) + * Persistence: optional, can be purged at any time, recommended ttl 30-60 days */ - ZT_STORED_OBJECT_NETWORK_CONFIG = 5, + ZT_STATE_OBJECT_PEER_IDENTITY = 4, /** - * Planet definition (object ID will be zero and should be ignored since there's only one) + * Network configuration + * + * Object ID: peer address + * Canonical path: /networks.d/.conf (16-digit hex ID) + * Persistence: required if network memberships should persist */ - ZT_STORED_OBJECT_PLANET = 6, + ZT_STATE_OBJECT_NETWORK_CONFIG = 5, /** - * Moon definition + * The planet (there is only one per... well... planet!) + * + * Object ID: world ID of planet, or 0 if unknown (first query) + * Canonical path: /planet + * Persistence: recommended */ - ZT_STORED_OBJECT_MOON = 7, + ZT_STATE_OBJECT_PLANET = 6, /** - * Multicast membership + * A moon (federated root set) + * + * Object ID: world ID of moon + * Canonical path: /moons.d/.moon (16-digit hex ID) + * Persistence: required if moon memberships should persist */ - ZT_STORED_OBJECT_MULTICAST_MEMBERSHIP = 8, + ZT_STATE_OBJECT_MOON = 7, /** - * IDs above this are never used by the core and are available for implementation use + * IDs above this value will not be used by the core (and could be used as implementation-specific IDs) */ - ZT_STORED_OBJECT__MAX_TYPE_ID = 255 + ZT_STATE_OBJECT__MAX_ID = 255 }; /** @@ -1221,59 +1264,38 @@ typedef void (*ZT_EventCallback)( const void *); /* Event payload (if applicable) */ /** - * Function to get an object from the data store - * - * Parameters: (1) object name, (2) buffer to fill, (3) size of buffer, (4) - * index in object to start reading, (5) result parameter that must be set - * to the actual size of the object if it exists. - * - * Object names can contain forward slash (/) path separators. They will - * never contain .. or backslash (\), so this is safe to map as a Unix-style - * path if the underlying storage permits. For security reasons we recommend - * returning errors if .. or \ are used. + * Callback for storing and/or publishing state information * - * The function must return the actual number of bytes read. If the object - * doesn't exist, it should return -1. -2 should be returned on other errors - * such as errors accessing underlying storage. + * See ZT_StateObjectType docs for information about each state object type + * and when and if it needs to be persisted. * - * If the read doesn't fit in the buffer, the max number of bytes should be - * read. The caller may call the function multiple times to read the whole - * object. + * An object of length -1 is sent to indicate that an object should be + * deleted. */ -typedef long (*ZT_DataStoreGetFunction)( +typedef void (*ZT_StatePutFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ void *, /* Thread ptr */ - const char *, - void *, - unsigned long, - unsigned long, - unsigned long *); + enum ZT_StateObjectType, /* State object type */ + uint64_t, /* State object ID (if applicable) */ + const void *, /* State object data */ + int); /* Length of data or -1 to delete */ /** - * Function to store an object in the data store + * Callback for retrieving stored state information * - * Parameters: (1) node, (2) user ptr, (3) object name, (4) object data, - * (5) object size, (6) secure? (bool). - * - * If secure is true, the file should be set readable and writable only - * to the user running ZeroTier One. What this means is platform-specific. - * - * Name semantics are the same as the get function. This must return zero on - * success. You can return any OS-specific error code on failure, as these - * may be visible in logs or error messages and might aid in debugging. - * - * If the data pointer is null, this must be interpreted as a delete - * operation. + * This function should return the number of bytes actually stored to the + * buffer or -1 if the state object was not found or the buffer was too + * small to store it. */ -typedef int (*ZT_DataStorePutFunction)( - ZT_Node *, - void *, +typedef int (*ZT_StateGetFunction)( + ZT_Node *, /* Node */ + void *, /* User ptr */ void *, /* Thread ptr */ - const char *, - const void *, - unsigned long, - int); + enum ZT_StateObjectType, /* State object type */ + uint64_t, /* State object ID (if applicable) */ + void *, /* Buffer to store state object data */ + unsigned int); /* Length of data buffer in bytes */ /** * Function to send a ZeroTier packet out over the wire @@ -1381,14 +1403,14 @@ struct ZT_Node_Callbacks long version; /** - * REQUIRED: Function to get objects from persistent storage + * REQUIRED: Function to store and/or replicate state objects */ - ZT_DataStoreGetFunction dataStoreGetFunction; + ZT_StatePutFunction statePutFunction; /** - * REQUIRED: Function to store objects in persistent storage + * REQUIRED: Function to retrieve state objects from an object store */ - ZT_DataStorePutFunction dataStorePutFunction; + ZT_StateGetFunction stateGetFunction; /** * REQUIRED: Function to send packets over the physical wire @@ -1449,6 +1471,49 @@ enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct */ void ZT_Node_delete(ZT_Node *node); +/** + * Notify node of an update to a state object + * + * This can be called after node startup to restore cached state objects such + * as network configurations for joined networks, planet, moons, etc. See + * the documentation of ZT_StateObjectType for more information. It's okay + * to call this for everything in the object store, but note that the node + * will automatically query for some core objects like identities so supplying + * these via this function is not necessary. + * + * Unless clustering is being implemented this function doesn't need to be + * used after startup. It could be called in response to filesystem changes + * to allow some degree of live configurability by filesystem observation. + * + * The return value of this function indicates whether the update was accepted + * as new. A return value of ZT_RESULT_OK indicates that the node gleaned new + * information from this update and that therefore (in cluster rumor mill mode) + * this update should be distributed to other members of a cluster. A return + * value of ZT_RESULT_OK_IGNORED indicates that the object did not provide any + * new information and therefore should not be propagated in a cluster. + * + * If clustering isn't being implemented the return value of this function can + * generally be ignored. + * + * ZT_RESULT_ERROR_BAD_PARAMETER can be returned if the parameter was invalid + * or not applicable. Object stores may delete the object in this case. + * + * @param node Node instance + * @param tptr Thread pointer to pass to functions/callbacks resulting from this call + * @param type State object type + * @param id State object ID + * @param data State object data + * @param len Length of state object data in bytes + * @return ZT_RESULT_OK if object was accepted or ZT_RESULT_OK_IGNORED if non-informative, error if object was invalid + */ +enum ZT_ResultCode ZT_Node_processStateUpdate( + ZT_Node *node, + void *tptr, + ZT_StateObjectType type, + uint64_t id, + const void *data, + unsigned int len); + /** * Process a packet received from the physical wire * diff --git a/node/Network.cpp b/node/Network.cpp index de2ea7d7..6dfb0b92 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -682,7 +682,7 @@ static _doZtFilterResult _doZtFilter( const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xffffffffffffULL),0); -Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr) : +Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr,const NetworkConfig *nconf) : RR(renv), _uPtr(uptr), _id(nwid), @@ -697,29 +697,11 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u for(int i=0;i *dconf = new Dictionary(); - NetworkConfig *nconf = new NetworkConfig(); - try { - std::string conf(RR->node->dataStoreGet(tPtr,confn)); - if (conf.length()) { - dconf->load(conf.c_str()); - if (nconf->fromDictionary(*dconf)) { - this->setConfiguration(tPtr,*nconf,false); - _lastConfigUpdate = 0; // we still want to re-request a new config from the network - gotConf = true; - } - } - } catch ( ... ) {} // ignore invalids, we'll re-request - delete nconf; - delete dconf; - - if (!gotConf) { - // Save a one-byte CR to persist membership while we request a real netconf - RR->node->dataStorePut(tPtr,confn,"\n",1,false); + if (nconf) { + this->setConfiguration(tPtr,*nconf,false); + _lastConfigUpdate = 0; // still want to re-request since it's likely outdated + } else { + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1); } if (!_portInitialized) { @@ -735,12 +717,9 @@ Network::~Network() ZT_VirtualNetworkConfig ctmp; _externalConfig(&ctmp); - char n[128]; if (_destroyed) { - // This is done in Node::leave() so we can pass tPtr + // This is done in Node::leave() so we can pass tPtr properly //RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); - RR->node->dataStoreDelete((void *)0,n); } else { RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp); } @@ -1188,10 +1167,8 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD if (saveToDisk) { Dictionary *d = new Dictionary(); try { - char n[64]; - Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); if (nconf.toDictionary(*d,false)) - RR->node->dataStorePut(tPtr,n,(const void *)d->data(),d->sizeBytes(),true); + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,_id,d->data(),d->sizeBytes()); } catch ( ... ) {} delete d; } diff --git a/node/Network.hpp b/node/Network.hpp index cce6c41f..454a3f20 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -88,8 +88,9 @@ public: * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param nwid Network ID * @param uptr Arbitrary pointer used by externally-facing API (for user use) + * @param nconf Network config, if known */ - Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr); + Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *uptr,const NetworkConfig *nconf); ~Network(); diff --git a/node/Node.cpp b/node/Node.cpp index bc4b858e..ccc63eed 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -33,6 +33,7 @@ #include "../version.h" #include "Constants.hpp" +#include "SharedPtr.hpp" #include "Node.hpp" #include "RuntimeEnvironment.hpp" #include "NetworkController.hpp" @@ -45,6 +46,7 @@ #include "Identity.hpp" #include "SelfAwareness.hpp" #include "Cluster.hpp" +#include "Network.hpp" const struct sockaddr_storage ZT_SOCKADDR_NULL = {0}; @@ -58,6 +60,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 _RR(this), RR(&_RR), _uPtr(uptr), + _networks(8), _now(now), _lastPingCheck(0), _lastHousekeepingRun(0) @@ -74,20 +77,31 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); - std::string idtmp(dataStoreGet(tptr,"identity.secret")); - if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { - TRACE("identity.secret not found, generating..."); - RR->identity.generate(); - idtmp = RR->identity.toString(true); - if (!dataStorePut(tptr,"identity.secret",idtmp,true)) - throw std::runtime_error("unable to write identity.secret"); + char tmp[512]; + std::string tmp2; + int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,0,tmp,sizeof(tmp) - 1); + if (n > 0) { + tmp[n] = (char)0; + if (!RR->identity.fromString(tmp)) + n = -1; } - RR->publicIdentityStr = RR->identity.toString(false); - RR->secretIdentityStr = RR->identity.toString(true); - idtmp = dataStoreGet(tptr,"identity.public"); - if (idtmp != RR->publicIdentityStr) { - if (!dataStorePut(tptr,"identity.public",RR->publicIdentityStr,false)) - throw std::runtime_error("unable to write identity.public"); + if (n <= 0) { + RR->identity.generate(); + tmp2 = RR->identity.toString(true); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length()); + tmp2 = RR->identity.toString(false); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length()); + } else { + n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp,sizeof(tmp) - 1); + if (n > 0) { + tmp[n] = (char)0; + if (RR->identity.toString(false) != tmp) + n = -1; + } + if (n <= 0) { + tmp2 = RR->identity.toString(false); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length()); + } } try { @@ -110,7 +124,7 @@ Node::~Node() { Mutex::Lock _l(_networks_m); - _networks.clear(); // ensure that networks are destroyed before shutdow + _networks.clear(); // destroy all networks before shutdown delete RR->sa; delete RR->topology; @@ -122,6 +136,88 @@ Node::~Node() #endif } +ZT_ResultCode Node::processStateUpdate( + void *tptr, + ZT_StateObjectType type, + uint64_t id, + const void *data, + unsigned int len) +{ + ZT_ResultCode r = ZT_RESULT_OK_IGNORED; + switch(type) { + + case ZT_STATE_OBJECT_PEER: { + } break; + + case ZT_STATE_OBJECT_NETWORK_CONFIG: + if (len <= (ZT_NETWORKCONFIG_DICT_CAPACITY - 1)) { + if (len < 2) { + Mutex::Lock _l(_networks_m); + SharedPtr &nw = _networks[id]; + if (!nw) + nw = SharedPtr(new Network(RR,tptr,id,(void *)0,(const NetworkConfig *)0)); + } else { + Dictionary *dict = new Dictionary(reinterpret_cast(data),len); + try { + NetworkConfig *nconf = new NetworkConfig(); + try { + if (nconf->fromDictionary(*dict)) { + Mutex::Lock _l(_networks_m); + SharedPtr &nw = _networks[id]; + if (nw) { + switch (nw->setConfiguration(tptr,*nconf,false)) { + default: + r = ZT_RESULT_ERROR_BAD_PARAMETER; + break; + case 1: + r = ZT_RESULT_OK_IGNORED; + break; + case 2: + r = ZT_RESULT_OK; + break; + } + } else { + nw = SharedPtr(new Network(RR,tptr,id,(void *)0,nconf)); + } + } else { + r = ZT_RESULT_ERROR_BAD_PARAMETER; + } + } catch ( ... ) { + r = ZT_RESULT_ERROR_BAD_PARAMETER; + } + delete nconf; + } catch ( ... ) { + r = ZT_RESULT_ERROR_BAD_PARAMETER; + } + delete dict; + } + } else { + r = ZT_RESULT_ERROR_BAD_PARAMETER; + } + break; + + case ZT_STATE_OBJECT_PLANET: + case ZT_STATE_OBJECT_MOON: + if (len <= ZT_WORLD_MAX_SERIALIZED_LENGTH) { + World w; + try { + w.deserialize(Buffer(data,len)); + if (( (w.type() == World::TYPE_MOON)&&(type == ZT_STATE_OBJECT_MOON) )||( (w.type() == World::TYPE_PLANET)&&(type == ZT_STATE_OBJECT_PLANET) )) { + r = (RR->topology->addWorld(tptr,w,false)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED; + } + } catch ( ... ) { + r = ZT_RESULT_ERROR_BAD_PARAMETER; + } + } else { + r = ZT_RESULT_ERROR_BAD_PARAMETER; + } + break; + + default: break; + } + return r; +} + ZT_ResultCode Node::processWirePacket( void *tptr, uint64_t now, @@ -311,7 +407,7 @@ ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr) Mutex::Lock _l(_networks_m); SharedPtr &nw = _networks[nwid]; if (!nw) - nw = SharedPtr(new Network(RR,tptr,nwid,uptr)); + nw = SharedPtr(new Network(RR,tptr,nwid,uptr,(const NetworkConfig *)0)); return ZT_RESULT_OK; } @@ -319,7 +415,6 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { ZT_VirtualNetworkConfig ctmp; void **nUserPtr = (void **)0; - { Mutex::Lock _l(_networks_m); SharedPtr *nw = _networks.get(nwid); @@ -330,12 +425,18 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) (*nw)->externalConfig(&ctmp); (*nw)->destroy(); nUserPtr = (*nw)->userPtr(); - _networks.erase(nwid); } if (nUserPtr) RR->node->configureVirtualNetworkPort(tptr,nwid,nUserPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + { + Mutex::Lock _l(_networks_m); + _networks.erase(nwid); + } + + RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid); + return ZT_RESULT_OK; } @@ -579,20 +680,6 @@ void Node::clusterStatus(ZT_ClusterStatus *cs) /* Node methods used only within node/ */ /****************************************************************************/ -std::string Node::dataStoreGet(void *tPtr,const char *name) -{ - char buf[1024]; - std::string r; - unsigned long olen = 0; - do { - long n = _cb.dataStoreGetFunction(reinterpret_cast(this),_uPtr,tPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen); - if (n <= 0) - return std::string(); - r.append(buf,n); - } while (r.length() < olen); - return r; -} - bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) { if (!Path::isAddressValidForPath(remoteAddress)) @@ -813,6 +900,23 @@ void ZT_Node_delete(ZT_Node *node) } catch ( ... ) {} } +enum ZT_ResultCode ZT_Node_processStateUpdate( + ZT_Node *node, + void *tptr, + ZT_StateObjectType type, + uint64_t id, + const void *data, + unsigned int len) +{ + try { + return reinterpret_cast(node)->processStateUpdate(tptr,type,id,data,len); + } catch (std::bad_alloc &exc) { + return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; + } catch ( ... ) { + return ZT_RESULT_FATAL_ERROR_INTERNAL; + } +} + enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, diff --git a/node/Node.hpp b/node/Node.hpp index 92e830c5..ceb3b000 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -82,6 +82,12 @@ public: // Public API Functions ---------------------------------------------------- + ZT_ResultCode processStateUpdate( + void *tptr, + ZT_StateObjectType type, + uint64_t id, + const void *data, + unsigned int len); ZT_ResultCode processWirePacket( void *tptr, uint64_t now, @@ -185,17 +191,16 @@ public: return _directPaths; } - inline bool dataStorePut(void *tPtr,const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,tPtr,name,data,len,(int)secure) == 0); } - inline bool dataStorePut(void *tPtr,const char *name,const std::string &data,bool secure) { return dataStorePut(tPtr,name,(const void *)data.data(),(unsigned int)data.length(),secure); } - inline void dataStoreDelete(void *tPtr,const char *name) { _cb.dataStorePutFunction(reinterpret_cast(this),_uPtr,tPtr,name,(const void *)0,0,0); } - std::string dataStoreGet(void *tPtr,const char *name); - inline void postEvent(void *tPtr,ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast(this),_uPtr,tPtr,ev,md); } inline int configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,tPtr,nwid,nuptr,op,nc); } inline bool online() const throw() { return _online; } + inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id,void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,maxlen); } + inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,(int)len); } + inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,(const void *)0,-1); } + #ifdef ZT_TRACE void postTrace(const char *module,unsigned int line,const char *fmt,...); #endif diff --git a/node/Topology.cpp b/node/Topology.cpp index 80f4ed4e..d4b424ff 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -68,15 +68,15 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : _trustedPathCount(0), _amRoot(false) { - try { - World cachedPlanet; - std::string buf(RR->node->dataStoreGet(tPtr,"planet")); - if (buf.length() > 0) { - Buffer dswtmp(buf.data(),(unsigned int)buf.length()); - cachedPlanet.deserialize(dswtmp,0); - } - addWorld(tPtr,cachedPlanet,false); - } catch ( ... ) {} + uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH]; + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,0,tmp,sizeof(tmp)); + if (n > 0) { + try { + World cachedPlanet; + cachedPlanet.deserialize(Buffer(tmp,(unsigned int)n),0); + addWorld(tPtr,cachedPlanet,false); + } catch ( ... ) {} // ignore invalid cached planets + } World defaultPlanet; { @@ -158,9 +158,8 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta) void Topology::saveIdentity(void *tPtr,const Identity &id) { if (id) { - char p[128]; - Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)id.address().toInt()); - RR->node->dataStorePut(tPtr,p,id.toString(false),false); + const std::string tmp(id.toString(false)); + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,id.address().toInt(),tmp.data(),(unsigned int)tmp.length()); } } @@ -327,19 +326,11 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew) return false; } - char savePath[64]; - if (existing->type() == World::TYPE_MOON) { - Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",existing->id()); - } else { - Utils::scopy(savePath,sizeof(savePath),"planet"); - } try { - Buffer dswtmp; - existing->serialize(dswtmp,false); - RR->node->dataStorePut(tPtr,savePath,dswtmp.data(),dswtmp.size(),false); - } catch ( ... ) { - RR->node->dataStoreDelete(tPtr,savePath); - } + Buffer sbuf; + existing->serialize(sbuf,false); + RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,existing->id(),sbuf.data(),sbuf.size()); + } catch ( ... ) {} _memoizeUpstreams(tPtr); @@ -348,21 +339,18 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew) void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed) { - char savePath[64]; - Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); - - try { - std::string moonBin(RR->node->dataStoreGet(tPtr,savePath)); - if (moonBin.length() > 1) { - Buffer wtmp(moonBin.data(),(unsigned int)moonBin.length()); + char tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH]; + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,id,tmp,sizeof(tmp)); + if (n > 0) { + try { World w; - w.deserialize(wtmp); + w.deserialize(Buffer(tmp,(unsigned int)n)); if ((w.type() == World::TYPE_MOON)&&(w.id() == id)) { addWorld(tPtr,w,true); return; } - } - } catch ( ... ) {} + } catch ( ... ) {} + } if (seed) { Mutex::Lock _l(_upstreams_m); @@ -381,9 +369,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id) if (m->id() != id) { nm.push_back(*m); } else { - char savePath[64]; - Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id); - RR->node->dataStoreDelete(tPtr,savePath); + RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,id); } } _moons.swap(nm); @@ -425,12 +411,12 @@ void Topology::clean(uint64_t now) Identity Topology::_getIdentity(void *tPtr,const Address &zta) { - char p[128]; - Utils::snprintf(p,sizeof(p),"iddb.d/%.10llx",(unsigned long long)zta.toInt()); - std::string ids(RR->node->dataStoreGet(tPtr,p)); - if (ids.length() > 0) { + char tmp[512]; + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,zta.toInt(),tmp,sizeof(tmp) - 1); + if (n > 0) { + tmp[n] = (char)0; try { - return Identity(ids); + return Identity(tmp); } catch ( ... ) {} // ignore invalid IDs } return Identity(); -- cgit v1.2.3 From aa06470cb6e779563e6c21e1166c36e30a2138ce Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 1 Jun 2017 20:32:43 -0700 Subject: More cleanup for cluster refactor. --- node/Node.cpp | 11 +++- service/OneService.cpp | 155 ++++++++++++------------------------------------- 2 files changed, 47 insertions(+), 119 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index ccc63eed..7421c467 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -146,8 +146,15 @@ ZT_ResultCode Node::processStateUpdate( ZT_ResultCode r = ZT_RESULT_OK_IGNORED; switch(type) { - case ZT_STATE_OBJECT_PEER: { - } break; + case ZT_STATE_OBJECT_PEER: + if (len) { + } + break; + + case ZT_STATE_OBJECT_PEER_IDENTITY: + if (len) { + } + break; case ZT_STATE_OBJECT_NETWORK_CONFIG: if (len <= (ZT_NETWORKCONFIG_DICT_CAPACITY - 1)) { diff --git a/service/OneService.cpp b/service/OneService.cpp index 7d290df7..5893a570 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -140,9 +140,6 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } // How often to check for new multicast subscriptions on a tap device #define ZT_TAP_CHECK_MULTICAST_INTERVAL 5000 -// Path under ZT1 home for controller database if controller is enabled -#define ZT_CONTROLLER_DB_PATH "controller.d" - // TCP fallback relay (run by ZeroTier, Inc. -- this will eventually go away) #define ZT_TCP_FALLBACK_RELAY "204.80.128.1/443" @@ -301,18 +298,12 @@ class OneServiceImpl; static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf); static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData); -static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize); -static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,const void *data,unsigned long len,int secure); +static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,const void *data,int len); +static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen); static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); - -#ifdef ZT_ENABLE_CLUSTER -static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len); -static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z); -#endif - static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); static int ShttpOnMessageBegin(http_parser *parser); @@ -379,9 +370,6 @@ struct TcpConnection Mutex writeBuf_m; }; -// Used to pseudo-randomize local source port picking -static volatile unsigned int _udpPortPickerCounter = 0; - class OneServiceImpl : public OneService { public: @@ -390,12 +378,17 @@ public: const std::string _homePath; std::string _authToken; std::string _controllerDbPath; + const std::string _iddbPath; + const std::string _networksPath; + const std::string _moonsPath; + EmbeddedNetworkController *_controller; Phy _phy; Node *_node; SoftwareUpdater *_updater; bool _updateAutoApply; unsigned int _primaryPort; + volatile unsigned int _udpPortPickerCounter; // Local configuration and memo-ized static path definitions json _localConfig; @@ -491,13 +484,17 @@ public: OneServiceImpl(const char *hp,unsigned int port) : _homePath((hp) ? hp : ".") - ,_controllerDbPath(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH) + ,_controllerDbPath(_homePath + ZT_PATH_SEPARATOR_S "controller.d") + ,_iddbPath(_homePath + ZT_PATH_SEPARATOR_S "iddb.d") + ,_networksPath(_homePath + ZT_PATH_SEPARATOR_S "networks.d") + ,_moonsPath(_homePath + ZT_PATH_SEPARATOR_S "moons.d") ,_controller((EmbeddedNetworkController *)0) ,_phy(this,false,true) ,_node((Node *)0) ,_updater((SoftwareUpdater *)0) ,_updateAutoApply(false) ,_primaryPort(port) + ,_udpPortPickerCounter(0) ,_v4TcpControlSocket((PhySocket *)0) ,_v6TcpControlSocket((PhySocket *)0) ,_lastDirectReceiveFromGlobal(0) @@ -568,15 +565,11 @@ public: _authToken = _trimString(_authToken); } - // Clean up any legacy files if present - OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S "peers.save").c_str()); - OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S "world").c_str()); - { struct ZT_Node_Callbacks cb; cb.version = 0; - cb.dataStoreGetFunction = SnodeDataStoreGetFunction; - cb.dataStorePutFunction = SnodeDataStorePutFunction; + cb.stateGetFunction = SnodeStateGetFunction; + cb.statePutFunction = SnodeStatePutFunction; cb.wirePacketSendFunction = SnodeWirePacketSendFunction; cb.virtualNetworkFrameFunction = SnodeVirtualNetworkFrameFunction; cb.virtualNetworkConfigFunction = SnodeVirtualNetworkConfigFunction; @@ -595,6 +588,7 @@ public: // Old style "trustedpaths" flat file -- will eventually go away FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r"); if (trustpaths) { + fprintf(stderr,"WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S); char buf[1024]; while ((fgets(buf,sizeof(buf),trustpaths))&&(trustedPathCount < ZT_MAX_TRUSTED_PATHS)) { int fno = 0; @@ -658,7 +652,7 @@ public: } applyLocalConfig(); - // Bind TCP control socket + // Bind TCP socket const int portTrials = (_primaryPort == 0) ? 256 : 1; // if port is 0, pick random for(int k=0;k 0) ? 0 : 0x7f000001)); // right now we just listen for TCP @127.0.0.1 in4.sin_port = Utils::hton((uint16_t)_primaryPort); _v4TcpControlSocket = _phy.tcpListen((const struct sockaddr *)&in4,this); @@ -679,8 +672,6 @@ public: memset((void *)&in6,0,sizeof(in6)); in6.sin6_family = AF_INET6; in6.sin6_port = in4.sin_port; - if (_allowManagementFrom.size() == 0) - in6.sin6_addr.s6_addr[15] = 1; // IPv6 localhost == ::1 _v6TcpControlSocket = _phy.tcpListen((const struct sockaddr *)&in6,this); // We must bind one of IPv4 or IPv6 -- support either failing to support hosts that @@ -706,7 +697,7 @@ public: return _termReason; } - // Write file containing primary port to be read by CLIs, etc. + // Save primary port to a file so CLIs and GUIs can learn it easily char portstr[64]; Utils::snprintf(portstr,sizeof(portstr),"%u",_ports[0]); OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr)); @@ -757,9 +748,11 @@ public: for(int i=0;i<3;++i) _portsBE[i] = Utils::hton((uint16_t)_ports[i]); + // Network controller is now enabled by default for desktop and server _controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str()); _node->setNetconfMaster((void *)_controller); +/* #ifdef ZT_ENABLE_CLUSTER if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S "cluster").c_str())) { _clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S "cluster").c_str()); @@ -808,7 +801,9 @@ public: } } #endif +*/ +/* { // Load existing networks std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str())); for(std::vector::iterator f(networksDotD.begin());f!=networksDotD.end();++f) { @@ -825,7 +820,9 @@ public: _node->orbit((void *)0,Utils::hexStrToU64(f->substr(0,dot).c_str()),0); } } +*/ + // Main I/O loop _nextBackgroundTaskDeadline = 0; uint64_t clockShouldBe = OSUtils::now(); _lastRestart = clockShouldBe; @@ -851,7 +848,7 @@ public: // Clean iddb.d on start and every 24 hours if ((now - lastCleanedIddb) > 86400000) { lastCleanedIddb = now; - OSUtils::cleanDirectory((_homePath + ZT_PATH_SEPARATOR_S "iddb.d").c_str(),now - ZT_IDDB_CLEANUP_AGE); + OSUtils::cleanDirectory(_iddbPath.c_str(),now - ZT_IDDB_CLEANUP_AGE); } // Attempt to detect sleep/wake events by detecting delay overruns @@ -1065,8 +1062,8 @@ public: return false; n->second.settings = settings; - char nlcpath[256]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + char nlcpath[4096]; + Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); FILE *out = fopen(nlcpath,"w"); if (out) { fprintf(out,"allowManaged=%d\n",(int)n->second.settings.allowManaged); @@ -1208,7 +1205,6 @@ public: settings["portMappingEnabled"] = false; // not supported in build #endif #ifndef ZT_SDK - settings["softwareUpdate"] = OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT); settings["softwareUpdateChannel"] = OSUtils::jsonString(settings["softwareUpdateChannel"],ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL); #endif @@ -1216,6 +1212,7 @@ public: res["planetWorldId"] = planet.id(); res["planetWorldTimestamp"] = planet.timestamp(); +/* #ifdef ZT_ENABLE_CLUSTER json cj; ZT_ClusterStatus cs; @@ -1242,6 +1239,7 @@ public: #else res["cluster"] = json(); #endif +*/ scode = 200; } else if (ps[0] == "moon") { @@ -1767,6 +1765,7 @@ public: inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) { +/* #ifdef ZT_ENABLE_CLUSTER if (sock == _clusterMessageSocket) { _lastDirectReceiveFromGlobal = OSUtils::now(); @@ -1774,6 +1773,7 @@ public: return; } #endif +*/ if ((len >= 16)&&(reinterpret_cast(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) _lastDirectReceiveFromGlobal = OSUtils::now(); @@ -1971,12 +1971,12 @@ public: if (sent > 0) { tc->lastActivity = OSUtils::now(); if ((unsigned long)sent >= (unsigned long)tc->writeBuf.length()) { - tc->writeBuf = ""; + tc->writeBuf.clear(); _phy.setNotifyWritable(sock,false); if (!tc->shouldKeepAlive) _phy.close(sock); // will call close handler to delete from _tcpConnections } else { - tc->writeBuf = tc->writeBuf.substr(sent); + tc->writeBuf.erase(tc->writeBuf.begin(),tc->writeBuf.begin() + sent); } } } else { @@ -2142,58 +2142,12 @@ public: } } - inline long nodeDataStoreGetFunction(const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize) + inline void nodeStatePutFunction(enum ZT_StateObjectType type,uint64_t id,const void *data,int len) { - std::string p(_dataStorePrepPath(name)); - if (!p.length()) - return -2; - - FILE *f = fopen(p.c_str(),"rb"); - if (!f) - return -1; - if (fseek(f,0,SEEK_END) != 0) { - fclose(f); - return -2; - } - long ts = ftell(f); - if (ts < 0) { - fclose(f); - return -2; - } - *totalSize = (unsigned long)ts; - if (fseek(f,(long)readIndex,SEEK_SET) != 0) { - fclose(f); - return -2; - } - long n = (long)fread(buf,1,bufSize,f); - fclose(f); - return n; } - inline int nodeDataStorePutFunction(const char *name,const void *data,unsigned long len,int secure) + inline int nodeStateGetFunction(enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen) { - std::string p(_dataStorePrepPath(name)); - if (!p.length()) - return -2; - - if (!data) { - OSUtils::rm(p.c_str()); - return 0; - } - - FILE *f = fopen(p.c_str(),"wb"); - if (!f) - return -1; - if (fwrite(data,len,1,f) == 1) { - fclose(f); - if (secure) - OSUtils::lockDownFile(p.c_str(),false); - return 0; - } else { - fclose(f); - OSUtils::rm(p.c_str()); - return -1; - } } inline int nodeWirePacketSendFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) @@ -2457,23 +2411,6 @@ public: return true; } - std::string _dataStorePrepPath(const char *name) const - { - std::string p(_homePath); - p.push_back(ZT_PATH_SEPARATOR); - char lastc = (char)0; - for(const char *n=name;(*n);++n) { - if ((*n == '.')&&(lastc == '.')) - return std::string(); // don't allow ../../ stuff as a precaution - if (*n == '/') { - OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - } else p.push_back(*n); - lastc = *n; - } - return p; - } - bool _trialBind(unsigned int port) { struct sockaddr_in in4; @@ -2514,10 +2451,10 @@ static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr { return reinterpret_cast(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); } static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData) { reinterpret_cast(uptr)->nodeEventCallback(event,metaData); } -static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize) -{ return reinterpret_cast(uptr)->nodeDataStoreGetFunction(name,buf,bufSize,readIndex,totalSize); } -static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,void *tptr,const char *name,const void *data,unsigned long len,int secure) -{ return reinterpret_cast(uptr)->nodeDataStorePutFunction(name,data,len,secure); } +static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,const void *data,int len) +{ reinterpret_cast(uptr)->nodeStatePutFunction(type,id,data,len); } +static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen) +{ return reinterpret_cast(uptr)->nodeStateGetFunction(type,id,data,maxlen); } static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) { return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); } static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) @@ -2526,22 +2463,6 @@ static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t z { return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr,localAddr,remoteAddr); } static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) { return reinterpret_cast(uptr)->nodePathLookupFunction(ztaddr,family,result); } - -#ifdef ZT_ENABLE_CLUSTER -static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len) -{ - OneServiceImpl *const impl = reinterpret_cast(uptr); - const ClusterDefinition::MemberDefinition &md = (*(impl->_clusterDefinition))[toMemberId]; - if (md.clusterEndpoint) - impl->_phy.udpSend(impl->_clusterMessageSocket,reinterpret_cast(&(md.clusterEndpoint)),data,len); -} -static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z) -{ - OneServiceImpl *const impl = reinterpret_cast(uptr); - return (int)(impl->_clusterDefinition->geo().locate(*(reinterpret_cast(addr)),*x,*y,*z)); -} -#endif - static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); } -- cgit v1.2.3 From 9b287392a4af95ee0d15db7e3d1f9dd6bd804060 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 5 Jun 2017 12:15:28 -0700 Subject: . --- include/ZeroTierOne.h | 49 ++- node/Buffer.hpp | 13 + node/Network.cpp | 21 +- node/Node.cpp | 4 +- node/Node.hpp | 2 + osdep/Binder.hpp | 10 + service/OneService.cpp | 813 ++++++++++++++++++++++++++++++++++--------------- 7 files changed, 627 insertions(+), 285 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 1e6f0ae8..9c295cee 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -297,7 +297,7 @@ enum ZT_ResultCode * @param x Result code * @return True if result code indicates a fatal error */ -#define ZT_ResultCode_isFatal(x) ((((int)(x)) > 0)&&(((int)(x)) < 1000)) +#define ZT_ResultCode_isFatal(x) ((((int)(x)) >= 100)&&(((int)(x)) < 1000)) /** * Status codes sent to status update callback when things happen @@ -393,6 +393,13 @@ enum ZT_Event /** * User message used with ZT_EVENT_USER_MESSAGE + * + * These are direct VL1 P2P messages for application use. Encryption and + * authentication in the ZeroTier protocol will guarantee the origin + * address and message content, but you are responsible for any other + * levels of authentication or access control that are required. Any node + * in the world can send you a user message! (Unless your network is air + * gapped.) */ typedef struct { @@ -720,24 +727,6 @@ typedef struct } v; } ZT_VirtualNetworkRule; -typedef struct -{ - /** - * 128-bit ID (GUID) of this capability - */ - uint64_t id[2]; - - /** - * Expiration time (measured vs. network config timestamp issued by controller) - */ - uint64_t expiration; - - struct { - uint64_t from; - uint64_t to; - } custody[ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH]; -} ZT_VirtualNetworkCapability; - /** * A route to be pushed on a virtual network */ @@ -1102,7 +1091,7 @@ enum ZT_StateObjectType ZT_STATE_OBJECT_NULL = 0, /** - * identity.public + * Public address and public key * * Object ID: this node's address if known, or 0 if unknown (first query) * Canonical path: /identity.public @@ -1111,10 +1100,10 @@ enum ZT_StateObjectType ZT_STATE_OBJECT_IDENTITY_PUBLIC = 1, /** - * identity.secret + * Full identity with secret key * * Object ID: this node's address if known, or 0 if unknown (first query) - * Canonical path: /identity.public + * Canonical path: /identity.secret * Persistence: required, should be stored with restricted permissions e.g. mode 0600 on *nix */ ZT_STATE_OBJECT_IDENTITY_SECRET = 2, @@ -1280,7 +1269,7 @@ typedef int (*ZT_StateGetFunction)( unsigned int); /* Length of data buffer in bytes */ /** - * Function to send a ZeroTier packet out over the wire + * Function to send a ZeroTier packet out over the physical wire (L2/L3) * * Parameters: * (1) Node @@ -1335,9 +1324,6 @@ typedef int (*ZT_WirePacketSendFunction)( * all configured ZeroTier interfaces and check to ensure that the supplied * addresses will not result in ZeroTier traffic being sent over a ZeroTier * interface (recursion). - * - * Obviously this is not required in configurations where this can't happen, - * such as network containers or embedded. */ typedef int (*ZT_PathCheckFunction)( ZT_Node *, /* Node */ @@ -1426,13 +1412,12 @@ struct ZT_Node_Callbacks }; /** - * Create a new ZeroTier One node - * - * Note that this can take a few seconds the first time it's called, as it - * will generate an identity. + * Create a new ZeroTier node * - * TODO: should consolidate function pointers into versioned structure for - * better API stability. + * This will attempt to load its identity via the state get function in the + * callback struct. If that fails it will generate a new identity and store + * it. Identity generation can take anywhere from a few hundred milliseconds + * to a few seconds depending on your CPU speed. * * @param node Result: pointer is set to new node instance on success * @param uptr User pointer to pass to functions/callbacks diff --git a/node/Buffer.hpp b/node/Buffer.hpp index fea32767..69ee1758 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -262,6 +262,19 @@ public: _b[_l++] = (char)c; } + /** + * Append secure random bytes + * + * @param n Number of random bytes to append + */ + inline void appendRandom(unsigned int n) + { + if (unlikely((_l + n) > C)) + throw std::out_of_range("Buffer: append beyond capacity"); + Utils::getSecureRandom(_b + _l,n); + _l += n; + } + /** * Append a C-array of bytes * diff --git a/node/Network.cpp b/node/Network.cpp index 6dfb0b92..74d81941 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -701,7 +701,26 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u this->setConfiguration(tPtr,*nconf,false); _lastConfigUpdate = 0; // still want to re-request since it's likely outdated } else { - RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1); + bool got = false; + Dictionary *dict = new Dictionary(); + try { + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1); + if (n > 1) { + NetworkConfig *nconf = new NetworkConfig(); + try { + if (nconf->fromDictionary(*dict)) { + this->setConfiguration(tPtr,*nconf,false); + _lastConfigUpdate = 0; // still want to re-request an update since it's likely outdated + got = true; + } + } catch ( ... ) {} + delete nconf; + } + } catch ( ... ) {} + delete dict; + + if (!got) + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1); } if (!_portInitialized) { diff --git a/node/Node.cpp b/node/Node.cpp index 7421c467..37586834 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -161,8 +161,10 @@ ZT_ResultCode Node::processStateUpdate( if (len < 2) { Mutex::Lock _l(_networks_m); SharedPtr &nw = _networks[id]; - if (!nw) + if (!nw) { nw = SharedPtr(new Network(RR,tptr,id,(void *)0,(const NetworkConfig *)0)); + r = ZT_RESULT_OK; + } } else { Dictionary *dict = new Dictionary(reinterpret_cast(data),len); try { diff --git a/node/Node.hpp b/node/Node.hpp index ceb3b000..f407c60c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -214,6 +214,8 @@ public: World planet() const; std::vector moons() const; + inline const Identity &identity() const { return _RR.identity; } + /** * Register that we are expecting a reply to a packet ID * diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index ee832825..fee1c3da 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -446,6 +446,16 @@ public: return aa; } + /** + * @param aa Vector to append local interface addresses to + */ + inline void allBoundLocalInterfaceAddresses(std::vector &aa) + { + Mutex::Lock _l(_lock); + for(std::vector<_Binding>::const_iterator i(_bindings.begin());i!=_bindings.end();++i) + aa.push_back(i->address); + } + private: std::vector<_Binding> _bindings; Mutex _lock; diff --git a/service/OneService.cpp b/service/OneService.cpp index 5893a570..6365131e 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -31,7 +31,6 @@ #include #include -#include #include #include #include @@ -47,6 +46,9 @@ #include "../node/MAC.hpp" #include "../node/Identity.hpp" #include "../node/World.hpp" +#include "../node/Salsa20.hpp" +#include "../node/Poly1305.hpp" +#include "../node/SHA512.hpp" #include "../osdep/Phy.hpp" #include "../osdep/Thread.hpp" @@ -155,10 +157,25 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } // Clean files from iddb.d that are older than this (60 days) #define ZT_IDDB_CLEANUP_AGE 5184000000ULL +// Maximum write buffer size for outgoing TCP connections (sanity limit) +#define ZT_TCP_MAX_WRITEQ_SIZE 33554432 + +// How often to check TCP connections and cluster links +#define ZT_TCP_CHECK_PERIOD 10000 + +// How often to send status info to cluster links +#define ZT_TCP_CLUSTER_SEND_STATUS_EVERY 30000 + +// TCP activity timeout +#define ZT_TCP_ACTIVITY_TIMEOUT 60000 + namespace ZeroTier { namespace { +// Fake TLS hello for TCP tunnel outgoing connections (TUNNELED mode) +static const char ZT_TCP_TUNNEL_HELLO[9] = { 0x17,0x03,0x03,0x00,0x04,(char)ZEROTIER_ONE_VERSION_MAJOR,(char)ZEROTIER_ONE_VERSION_MINOR,(char)((ZEROTIER_ONE_VERSION_REVISION >> 8) & 0xff),(char)(ZEROTIER_ONE_VERSION_REVISION & 0xff) }; + static std::string _trimString(const std::string &s) { unsigned long end = (unsigned long)s.length(); @@ -342,32 +359,53 @@ static const struct http_parser_settings HTTP_PARSER_SETTINGS = { }; #endif +/** + * A TCP connection and related state and buffers + */ struct TcpConnection { enum { + TCP_UNCATEGORIZED_INCOMING, // uncategorized incoming connection TCP_HTTP_INCOMING, - TCP_HTTP_OUTGOING, // not currently used - TCP_TUNNEL_OUTGOING // fale-SSL outgoing tunnel -- HTTP-related fields are not used + TCP_HTTP_OUTGOING, + TCP_TUNNEL_OUTGOING, // TUNNELED mode proxy outbound connection + TCP_CLUSTER_BACKPLANE, } type; - bool shouldKeepAlive; OneServiceImpl *parent; PhySocket *sock; InetAddress from; + unsigned long lastReceive; + + // Used for inbound HTTP connections http_parser parser; unsigned long messageSize; - uint64_t lastActivity; - std::string currentHeaderField; std::string currentHeaderValue; - std::string url; std::string status; std::map< std::string,std::string > headers; - std::string body; - std::string writeBuf; - Mutex writeBuf_m; + // Used for cluster backplane connections + uint64_t clusterMemberId; + unsigned int clusterMemberVersionMajor; + unsigned int clusterMemberVersionMinor; + unsigned int clusterMemberVersionRev; + std::vector< InetAddress > clusterMemberLocalAddresses; + + std::string readq; + std::string writeq; + Mutex writeq_m; +}; + +/** + * Message types for cluster backplane communication + */ +enum ClusterMessageType +{ + CLUSTER_MESSAGE_STATUS = 0, + CLUSTER_MESSAGE_STATE_OBJECT = 1, + CLUSTER_MESSAGE_PROXY_SEND = 2 }; class OneServiceImpl : public OneService @@ -389,8 +427,10 @@ public: bool _updateAutoApply; unsigned int _primaryPort; volatile unsigned int _udpPortPickerCounter; + uint64_t _clusterMemberId; + uint8_t _clusterKey[32]; // secret key for cluster backplane config - // Local configuration and memo-ized static path definitions + // Local configuration and memo-ized information from it json _localConfig; Hashtable< uint64_t,std::vector > _v4Hints; Hashtable< uint64_t,std::vector > _v6Hints; @@ -400,6 +440,7 @@ public: std::vector< InetAddress > _globalV6Blacklist; std::vector< InetAddress > _allowManagementFrom; std::vector< std::string > _interfacePrefixBlacklist; + std::vector< InetAddress > _clusterBackplaneAddresses; Mutex _localConfig_m; /* @@ -417,6 +458,10 @@ public: unsigned int _ports[3]; uint16_t _portsBE[3]; // ports in big-endian network byte order as in sockaddr + // Local interface addresses obtained from bindings + std::vector _localInterfaceAddresses; + Mutex _localInterfaceAddresses_m; + // Sockets for JSON API -- bound only to V4 and V6 localhost PhySocket *_v4TcpControlSocket; PhySocket *_v6TcpControlSocket; @@ -455,7 +500,8 @@ public: Mutex _nets_m; // Active TCP/IP connections - std::set< TcpConnection * > _tcpConnections; // no mutex for this since it's done in the main loop thread only + std::vector< TcpConnection * > _tcpConnections; + Mutex _tcpConnections_m; TcpConnection *_tcpFallbackTunnel; // Termination status information @@ -469,13 +515,6 @@ public: PortMapper *_portMapper; #endif - // Cluster management instance if enabled -#ifdef ZT_ENABLE_CLUSTER - PhySocket *_clusterMessageSocket; - ClusterDefinition *_clusterDefinition; - unsigned int _clusterMemberId; -#endif - // Set to false to force service to stop volatile bool _run; Mutex _run_m; @@ -512,7 +551,6 @@ public: #ifdef ZT_ENABLE_CLUSTER ,_clusterMessageSocket((PhySocket *)0) ,_clusterDefinition((ClusterDefinition *)0) - ,_clusterMemberId(0) #endif ,_run(true) { @@ -822,6 +860,14 @@ public: } */ + { + uint8_t tmp[64]; + SHA512::hash(tmp,_node->identity().privateKeyPair().priv.data,ZT_C25519_PRIVATE_KEY_LEN); + memcpy(_clusterKey,tmp,32); + } + _clusterMemberId = _node->prng(); + if (!_clusterMemberId) _clusterMemberId = 1; + // Main I/O loop _nextBackgroundTaskDeadline = 0; uint64_t clockShouldBe = OSUtils::now(); @@ -911,6 +957,7 @@ public: _node->clearLocalInterfaceAddresses(); + // Tell Node about uPnP and NAT-PMP bound external addresses #ifdef ZT_USE_MINIUPNPC if (_portMapper) { std::vector mappedAddresses(_portMapper->get()); @@ -919,9 +966,20 @@ public: } #endif + // Tell Node about local interface addresses bound to the primary port std::vector boundAddrs(_bindings[0].allBoundLocalInterfaceAddresses()); for(std::vector::const_iterator i(boundAddrs.begin());i!=boundAddrs.end();++i) _node->addLocalInterfaceAddress(reinterpret_cast(&(*i))); + + // Memoize all local interface addresses for use in clustering -- we tell other cluster members about these + { + Mutex::Lock _l(_localInterfaceAddresses_m); + _localInterfaceAddresses.clear(); + for(int i=0;i<3;++i) { + if (_ports[i] > 0) + _bindings[i].allBoundLocalInterfaceAddresses(_localInterfaceAddresses); + } + } } const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 100; @@ -939,6 +997,7 @@ public: } try { + Mutex::Lock _l(_tcpConnections_m); while (!_tcpConnections.empty()) _phy.close((*_tcpConnections.begin())->sock); } catch ( ... ) {} @@ -980,35 +1039,35 @@ public: } #ifdef ZT_SDK - virtual void leave(const char *hp) - { - _node->leave(Utils::hexStrToU64(hp),NULL,NULL); - } + virtual void leave(const char *hp) + { + _node->leave(Utils::hexStrToU64(hp),NULL,NULL); + } virtual void join(const char *hp) { _node->join(Utils::hexStrToU64(hp),NULL,NULL); } - virtual std::string givenHomePath() - { - return _homePath; - } + virtual std::string givenHomePath() + { + return _homePath; + } - virtual EthernetTap * getTap(uint64_t nwid) - { + virtual EthernetTap * getTap(uint64_t nwid) + { Mutex::Lock _l(_nets_m); std::map::const_iterator n(_nets.find(nwid)); if (n == _nets.end()) - return NULL; + return NULL; return n->second.tap; - } + } - virtual EthernetTap *getTap(InetAddress &addr) - { - Mutex::Lock _l(_nets_m); + virtual EthernetTap *getTap(InetAddress &addr) + { + Mutex::Lock _l(_nets_m); std::map::iterator it; - for(it = _nets.begin(); it != _nets.end(); it++) { + for(it = _nets.begin(); it != _nets.end(); it++) { if(it->second.tap) { for(int j=0; jsecond.tap->_ips.size(); j++) { if(it->second.tap->_ips[j].isEqualPrefix(addr) || it->second.tap->_ips[j].ipsEqual(addr)) { @@ -1016,9 +1075,9 @@ public: } } } - } - return NULL; - } + } + return NULL; + } virtual Node * getNode() { @@ -1029,9 +1088,8 @@ public: { Mutex::Lock _l(_nets_m); std::map::iterator i; - for(i = _nets.begin(); i != _nets.end(); i++) { - delete i->second.tap; - } + for(i = _nets.begin(); i != _nets.end(); i++) + delete i->second.tap; } #endif // ZT_SDK @@ -1078,7 +1136,9 @@ public: return true; } - // Internal implementation methods ----------------------------------------- + // ========================================================================= + // Internal implementation methods for control plane, route setup, etc. + // ========================================================================= inline unsigned int handleControlPlaneHttpRequest( const InetAddress &fromAddress, @@ -1759,22 +1819,118 @@ public: } } + void announceStatusToClusterMember(TcpConnection *tc) + { + Buffer<4096> buf; + + buf.appendRandom(16); + buf.addSize(8); // space for MAC + buf.append((uint8_t)CLUSTER_MESSAGE_STATUS); + buf.append(_clusterMemberId); + buf.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR); + buf.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR); + buf.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); + + { + Mutex::Lock _l(_localInterfaceAddresses_m); + buf.append((uint16_t)_localInterfaceAddresses.size()); + for(std::vector::const_iterator i(_localInterfaceAddresses.begin());i!=_localInterfaceAddresses.end();++i) { + i->serialize(buf); + if ((buf.size() + 32) > buf.capacity()) + break; + } + } + + Mutex::Lock _l(tc->writeq_m); + + if (tc->writeq.length() == 0) + _phy.setNotifyWritable(tc->sock,true); + + const unsigned int mlen = buf.size(); + tc->writeq.push_back((char)((mlen >> 16) & 0xff)); + tc->writeq.push_back((char)((mlen >> 8) & 0xff)); + tc->writeq.push_back((char)(mlen & 0xff)); + + char *data = reinterpret_cast(buf.unsafeData()); + + uint8_t key[32]; + memcpy(key,_clusterKey,32); + for(int i=0;i<8;++i) key[i] ^= data[i]; + Salsa20 s20(key,data + 8); + + uint8_t macKey[32]; + uint8_t mac[16]; + memset(macKey,0,32); + s20.crypt12(macKey,macKey,32); + s20.crypt12(data + 24,data + 24,mlen - 24); + Poly1305::compute(mac,data + 24,mlen - 24,macKey); + memcpy(data + 16,mac,8); + + tc->writeq.append(data,mlen); + } + + void replicateStateObjectToCluster(const ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len,const uint64_t everyoneBut) + { + uint8_t *buf = new uint8_t[len + 34]; + try { + std::vector sentTo; + if (everyoneBut) + sentTo.push_back(everyoneBut); + Mutex::Lock _l(_tcpConnections_m); + for(std::vector::const_iterator ci(_tcpConnections.begin());ci!=_tcpConnections.end();++ci) { + TcpConnection *const c = *ci; + if ((c->type == TcpConnection::TCP_CLUSTER_BACKPLANE)&&(c->clusterMemberId != 0)&&(std::find(sentTo.begin(),sentTo.end(),c->clusterMemberId) == sentTo.end())) { + sentTo.push_back(c->clusterMemberId); + Mutex::Lock _l2(c->writeq_m); + + if (c->writeq.length() == 0) + _phy.setNotifyWritable(c->sock,true); + + const unsigned int mlen = len + 34; + c->writeq.push_back((char)((mlen >> 16) & 0xff)); + c->writeq.push_back((char)((mlen >> 8) & 0xff)); + c->writeq.push_back((char)(mlen & 0xff)); + + Utils::getSecureRandom(buf,16); + + buf[24] = (uint8_t)CLUSTER_MESSAGE_STATE_OBJECT; + buf[25] = (uint8_t)type; + buf[26] = (uint8_t)((id >> 56) & 0xff); + buf[27] = (uint8_t)((id >> 48) & 0xff); + buf[28] = (uint8_t)((id >> 40) & 0xff); + buf[29] = (uint8_t)((id >> 32) & 0xff); + buf[30] = (uint8_t)((id >> 24) & 0xff); + buf[31] = (uint8_t)((id >> 16) & 0xff); + buf[32] = (uint8_t)((id >> 8) & 0xff); + buf[33] = (uint8_t)(id & 0xff); + memcpy(buf + 34,data,len); + + uint8_t key[32]; + memcpy(key,_clusterKey,32); + for(int i=0;i<8;++i) key[i] ^= buf[i]; + Salsa20 s20(key,buf + 8); + + uint8_t macKey[32]; + uint8_t mac[16]; + memset(macKey,0,32); + s20.crypt12(macKey,macKey,32); + s20.crypt12(buf + 24,buf + 24,mlen - 24); + Poly1305::compute(mac,buf + 24,mlen - 24,macKey); + memcpy(buf + 16,mac,8); + + c->writeq.append(reinterpret_cast(buf),len + 34); + } + } + } catch ( ... ) {} // sanity check + delete [] buf; + } + // ========================================================================= // Handlers for Node and Phy<> callbacks // ========================================================================= inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) { -/* -#ifdef ZT_ENABLE_CLUSTER - if (sock == _clusterMessageSocket) { - _lastDirectReceiveFromGlobal = OSUtils::now(); - _node->clusterHandleIncomingMessage(data,len); - return; - } -#endif -*/ - if ((len >= 16)&&(reinterpret_cast(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) _lastDirectReceiveFromGlobal = OSUtils::now(); @@ -1798,38 +1954,27 @@ public: inline void phyOnTcpConnect(PhySocket *sock,void **uptr,bool success) { - if (!success) + if (!success) { + phyOnTcpClose(sock,uptr); return; + } - // Outgoing TCP connections are always TCP fallback tunnel connections. - - TcpConnection *tc = new TcpConnection(); - _tcpConnections.insert(tc); - - tc->type = TcpConnection::TCP_TUNNEL_OUTGOING; - tc->shouldKeepAlive = true; - tc->parent = this; + TcpConnection *const tc = reinterpret_cast(*uptr); + if (!tc) { // sanity check + _phy.close(sock,true); + return; + } tc->sock = sock; - // from and parser are not used - tc->messageSize = 0; // unused - tc->lastActivity = OSUtils::now(); - // HTTP stuff is not used - tc->writeBuf = ""; - *uptr = (void *)tc; - - // Send "hello" message - tc->writeBuf.push_back((char)0x17); - tc->writeBuf.push_back((char)0x03); - tc->writeBuf.push_back((char)0x03); // fake TLS 1.2 header - tc->writeBuf.push_back((char)0x00); - tc->writeBuf.push_back((char)0x04); // mlen == 4 - tc->writeBuf.push_back((char)ZEROTIER_ONE_VERSION_MAJOR); - tc->writeBuf.push_back((char)ZEROTIER_ONE_VERSION_MINOR); - tc->writeBuf.push_back((char)((ZEROTIER_ONE_VERSION_REVISION >> 8) & 0xff)); - tc->writeBuf.push_back((char)(ZEROTIER_ONE_VERSION_REVISION & 0xff)); - _phy.setNotifyWritable(sock,true); - - _tcpFallbackTunnel = tc; + + if (tc->type == TcpConnection::TCP_TUNNEL_OUTGOING) { + if (_tcpFallbackTunnel) + _phy.close(_tcpFallbackTunnel->sock); + _tcpFallbackTunnel = tc; + _phy.streamSend(sock,ZT_TCP_TUNNEL_HELLO,sizeof(ZT_TCP_TUNNEL_HELLO)); + } else if (tc->type == TcpConnection::TCP_CLUSTER_BACKPLANE) { + } else { + _phy.close(sock,true); + } } inline void phyOnTcpAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from) @@ -1839,149 +1984,313 @@ public: return; } else { TcpConnection *tc = new TcpConnection(); - _tcpConnections.insert(tc); - tc->type = TcpConnection::TCP_HTTP_INCOMING; - tc->shouldKeepAlive = true; + { + Mutex::Lock _l(_tcpConnections_m); + _tcpConnections.push_back(tc); + } + + tc->type = TcpConnection::TCP_UNCATEGORIZED_INCOMING; tc->parent = this; tc->sock = sockN; tc->from = from; + tc->lastReceive = OSUtils::now(); http_parser_init(&(tc->parser),HTTP_REQUEST); tc->parser.data = (void *)tc; tc->messageSize = 0; - tc->lastActivity = OSUtils::now(); - tc->currentHeaderField = ""; - tc->currentHeaderValue = ""; - tc->url = ""; - tc->status = ""; - tc->headers.clear(); - tc->body = ""; - tc->writeBuf = ""; + *uptrN = (void *)tc; } } - inline void phyOnTcpClose(PhySocket *sock,void **uptr) + void phyOnTcpClose(PhySocket *sock,void **uptr) { TcpConnection *tc = (TcpConnection *)*uptr; if (tc) { - if (tc == _tcpFallbackTunnel) + if (tc == _tcpFallbackTunnel) { _tcpFallbackTunnel = (TcpConnection *)0; - _tcpConnections.erase(tc); + } + { + Mutex::Lock _l(_tcpConnections_m); + _tcpConnections.erase(std::remove(_tcpConnections.begin(),_tcpConnections.end(),tc),_tcpConnections.end()); + } delete tc; } } - inline void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) + void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) { - TcpConnection *tc = reinterpret_cast(*uptr); - switch(tc->type) { + try { + if (!len) return; // sanity check, should never happen + TcpConnection *tc = reinterpret_cast(*uptr); + tc->lastReceive = OSUtils::now(); + switch(tc->type) { + + case TcpConnection::TCP_UNCATEGORIZED_INCOMING: + switch(reinterpret_cast(data)[0]) { + // 0x93 is first byte of cluster backplane connections + case 0x93: { + bool allow = false; + { + Mutex::Lock _l(_localConfig_m); + for(std::vector< InetAddress >::const_iterator i(_clusterBackplaneAddresses.begin());i!=_clusterBackplaneAddresses.end();++i) { + if (tc->from.ipsEqual(*i)) { + allow = true; + break; + } + } + } + if (allow) { // note that we also auth each packet cryptographically -- this is just a first line sanity check + tc->type = TcpConnection::TCP_CLUSTER_BACKPLANE; + tc->clusterMemberId = 0; // unknown, waiting for first status message + announceStatusToClusterMember(tc); + if (len > 1) + phyOnTcpData(sock,uptr,reinterpret_cast(data) + 1,len - 1); + } else { + _phy.close(sock); + } + } break; + + // HTTP: GET, PUT, POST, HEAD + case 'G': + case 'P': + case 'H': { + bool allow; + { + Mutex::Lock _l(_localConfig_m); + if (_allowManagementFrom.size() == 0) { + allow = (tc->from.ipScope() == InetAddress::IP_SCOPE_LOOPBACK); + } else { + allow = false; + for(std::vector::const_iterator i(_allowManagementFrom.begin());i!=_allowManagementFrom.end();++i) { + if (i->containsAddress(tc->from)) { + allow = true; + break; + } + } + } + } + if (allow) { + tc->type = TcpConnection::TCP_HTTP_INCOMING; + phyOnTcpData(sock,uptr,data,len); + } else { + _phy.close(sock); + } + } break; + + // Drop unknown protocols + default: + _phy.close(sock); + break; + } + return; - case TcpConnection::TCP_HTTP_INCOMING: - case TcpConnection::TCP_HTTP_OUTGOING: - http_parser_execute(&(tc->parser),&HTTP_PARSER_SETTINGS,(const char *)data,len); - if ((tc->parser.upgrade)||(tc->parser.http_errno != HPE_OK)) { - _phy.close(sock); + case TcpConnection::TCP_HTTP_INCOMING: + case TcpConnection::TCP_HTTP_OUTGOING: + http_parser_execute(&(tc->parser),&HTTP_PARSER_SETTINGS,(const char *)data,len); + if ((tc->parser.upgrade)||(tc->parser.http_errno != HPE_OK)) + _phy.close(sock); return; - } - break; - case TcpConnection::TCP_TUNNEL_OUTGOING: - tc->body.append((const char *)data,len); - while (tc->body.length() >= 5) { - const char *data = tc->body.data(); - const unsigned long mlen = ( ((((unsigned long)data[3]) & 0xff) << 8) | (((unsigned long)data[4]) & 0xff) ); - if (tc->body.length() >= (mlen + 5)) { - InetAddress from; + case TcpConnection::TCP_TUNNEL_OUTGOING: + tc->readq.append((const char *)data,len); + while (tc->readq.length() >= 5) { + const char *data = tc->readq.data(); + const unsigned long mlen = ( ((((unsigned long)data[3]) & 0xff) << 8) | (((unsigned long)data[4]) & 0xff) ); + if (tc->readq.length() >= (mlen + 5)) { + InetAddress from; unsigned long plen = mlen; // payload length, modified if there's an IP header - data += 5; // skip forward past pseudo-TLS junk and mlen - if (plen == 4) { - // Hello message, which isn't sent by proxy and would be ignored by client - } else if (plen) { - // Messages should contain IPv4 or IPv6 source IP address data - switch(data[0]) { - case 4: // IPv4 - if (plen >= 7) { - from.set((const void *)(data + 1),4,((((unsigned int)data[5]) & 0xff) << 8) | (((unsigned int)data[6]) & 0xff)); - data += 7; // type + 4 byte IP + 2 byte port - plen -= 7; - } else { + data += 5; // skip forward past pseudo-TLS junk and mlen + if (plen == 4) { + // Hello message, which isn't sent by proxy and would be ignored by client + } else if (plen) { + // Messages should contain IPv4 or IPv6 source IP address data + switch(data[0]) { + case 4: // IPv4 + if (plen >= 7) { + from.set((const void *)(data + 1),4,((((unsigned int)data[5]) & 0xff) << 8) | (((unsigned int)data[6]) & 0xff)); + data += 7; // type + 4 byte IP + 2 byte port + plen -= 7; + } else { + _phy.close(sock); + return; + } + break; + case 6: // IPv6 + if (plen >= 19) { + from.set((const void *)(data + 1),16,((((unsigned int)data[17]) & 0xff) << 8) | (((unsigned int)data[18]) & 0xff)); + data += 19; // type + 16 byte IP + 2 byte port + plen -= 19; + } else { + _phy.close(sock); + return; + } + break; + case 0: // none/omitted + ++data; + --plen; + break; + default: // invalid address type _phy.close(sock); return; - } - break; - case 6: // IPv6 - if (plen >= 19) { - from.set((const void *)(data + 1),16,((((unsigned int)data[17]) & 0xff) << 8) | (((unsigned int)data[18]) & 0xff)); - data += 19; // type + 16 byte IP + 2 byte port - plen -= 19; - } else { + } + + if (from) { + InetAddress fakeTcpLocalInterfaceAddress((uint32_t)0xffffffff,0xffff); + const ZT_ResultCode rc = _node->processWirePacket( + (void *)0, + OSUtils::now(), + reinterpret_cast(&fakeTcpLocalInterfaceAddress), + reinterpret_cast(&from), + data, + plen, + &_nextBackgroundTaskDeadline); + if (ZT_ResultCode_isFatal(rc)) { + char tmp[256]; + Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + Mutex::Lock _l(_termReason_m); + _termReason = ONE_UNRECOVERABLE_ERROR; + _fatalErrorMessage = tmp; + this->terminate(); _phy.close(sock); return; } + } + } + + if (tc->readq.length() > (mlen + 5)) + tc->readq.erase(tc->readq.begin(),tc->readq.begin() + (mlen + 5)); + else tc->readq.clear(); + } else break; + } + return; + + case TcpConnection::TCP_CLUSTER_BACKPLANE: + tc->readq.append((const char *)data,len); + if (tc->readq.length() >= 28) { // got 3-byte message size + 16-byte IV + 8-byte MAC + 1-byte type (encrypted) + uint8_t *data = reinterpret_cast(const_cast(tc->readq.data())); + unsigned long mlen = ( ((unsigned long)data[0] << 16) | ((unsigned long)data[1] << 8) | (unsigned long)data[2] ); + if ((mlen < 25)||(mlen > ZT_TCP_MAX_WRITEQ_SIZE)) { + _phy.close(sock); + return; + } else if (tc->readq.length() >= (mlen + 3)) { // got entire message + data += 3; + + uint8_t key[32]; + memcpy(key,_clusterKey,32); + for(int i=0;i<8;++i) key[i] ^= data[i]; // first 8 bytes of IV get XORed with key + Salsa20 s20(key,data + 8); // last 8 bytes of IV are fed into Salsa20 directly as its 64-bit IV + + uint8_t macKey[32]; + uint8_t mac[16]; + memset(macKey,0,32); + s20.crypt12(macKey,macKey,32); + Poly1305::compute(mac,data + 24,mlen - 24,macKey); + if (!Utils::secureEq(mac,data + 16,8)) { + _phy.close(sock); + return; + } + s20.crypt12(data + 24,data + 24,mlen - 24); + + switch((ClusterMessageType)data[24]) { + case CLUSTER_MESSAGE_STATUS: + if (mlen > (25 + 16)) { + Buffer<4096> tmp(data + 25,mlen - 25); + try { + tc->clusterMemberId = tmp.at(0); + if (tc->clusterMemberId == _clusterMemberId) { // shouldn't happen, but don't allow self-to-self + _phy.close(sock); + return; + } + tc->clusterMemberVersionMajor = tmp.at(8); + tc->clusterMemberVersionMinor = tmp.at(10); + tc->clusterMemberVersionRev = tmp.at(12); + const unsigned int clusterMemberLocalAddressCount = tmp.at(14); + std::vector la; + unsigned int ptr = 16; + for(unsigned int k=0;kclusterMemberLocalAddresses.swap(la); + } catch ( ... ) {} + } + break; + + case CLUSTER_MESSAGE_STATE_OBJECT: + if (mlen >= (25 + 9)) { // type + object ID + [data] + const uint64_t objId = ( + ((uint64_t)data[26] << 56) | + ((uint64_t)data[27] << 48) | + ((uint64_t)data[28] << 40) | + ((uint64_t)data[29] << 32) | + ((uint64_t)data[30] << 24) | + ((uint64_t)data[31] << 16) | + ((uint64_t)data[32] << 8) | + (uint64_t)data[33] + ); + if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34)) == ZT_RESULT_OK) + replicateStateObjectToCluster((ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34),tc->clusterMemberId); + } break; - case 0: // none/omitted - ++data; - --plen; + + case CLUSTER_MESSAGE_PROXY_SEND: + if (mlen > 25) { + Buffer<4096> tmp(data + 25,mlen - 25); + try { + InetAddress dest,src; + unsigned int ptr = dest.deserialize(tmp); + ptr += src.deserialize(tmp,ptr); + if (ptr < tmp.size()) { + bool local; + { + Mutex::Lock _l(_localInterfaceAddresses_m); + local = (std::find(_localInterfaceAddresses.begin(),_localInterfaceAddresses.end(),src) != _localInterfaceAddresses.end()); + } + if (local) + nodeWirePacketSendFunction(&src,&dest,reinterpret_cast(tmp.data()) + ptr,tmp.size() - ptr,0); + } + } catch ( ... ) {} + } break; - default: // invalid address type - _phy.close(sock); - return; } - if (from) { - InetAddress fakeTcpLocalInterfaceAddress((uint32_t)0xffffffff,0xffff); - const ZT_ResultCode rc = _node->processWirePacket( - (void *)0, - OSUtils::now(), - reinterpret_cast(&fakeTcpLocalInterfaceAddress), - reinterpret_cast(&from), - data, - plen, - &_nextBackgroundTaskDeadline); - if (ZT_ResultCode_isFatal(rc)) { - char tmp[256]; - Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); - Mutex::Lock _l(_termReason_m); - _termReason = ONE_UNRECOVERABLE_ERROR; - _fatalErrorMessage = tmp; - this->terminate(); - _phy.close(sock); - return; - } - } + tc->readq.erase(tc->readq.begin(),tc->readq.begin() + mlen); } + } + return; - if (tc->body.length() > (mlen + 5)) - tc->body = tc->body.substr(mlen + 5); - else tc->body = ""; - } else break; - } - break; - + } + } catch ( ... ) { + _phy.close(sock); } } inline void phyOnTcpWritable(PhySocket *sock,void **uptr) { TcpConnection *tc = reinterpret_cast(*uptr); - Mutex::Lock _l(tc->writeBuf_m); - if (tc->writeBuf.length() > 0) { - long sent = (long)_phy.streamSend(sock,tc->writeBuf.data(),(unsigned long)tc->writeBuf.length(),true); - if (sent > 0) { - tc->lastActivity = OSUtils::now(); - if ((unsigned long)sent >= (unsigned long)tc->writeBuf.length()) { - tc->writeBuf.clear(); - _phy.setNotifyWritable(sock,false); - if (!tc->shouldKeepAlive) - _phy.close(sock); // will call close handler to delete from _tcpConnections - } else { - tc->writeBuf.erase(tc->writeBuf.begin(),tc->writeBuf.begin() + sent); + bool closeit = false; + { + Mutex::Lock _l(tc->writeq_m); + if (tc->writeq.length() > 0) { + long sent = (long)_phy.streamSend(sock,tc->writeq.data(),(unsigned long)tc->writeq.length(),true); + if (sent > 0) { + if ((unsigned long)sent >= (unsigned long)tc->writeq.length()) { + tc->writeq.clear(); + _phy.setNotifyWritable(sock,false); + + if (tc->type == TcpConnection::TCP_HTTP_INCOMING) + closeit = true; // HTTP keep alive not supported + } else { + tc->writeq.erase(tc->writeq.begin(),tc->writeq.begin() + sent); + } } + } else { + _phy.setNotifyWritable(sock,false); } - } else { - _phy.setNotifyWritable(sock,false); } + if (closeit) + _phy.close(sock); } inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {} @@ -2148,6 +2457,27 @@ public: inline int nodeStateGetFunction(enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen) { + char p[4096]; + FILE *f = (FILE *)0; + switch(type) { + case ZT_STATE_OBJECT_IDENTITY_PUBLIC: + break; + case ZT_STATE_OBJECT_IDENTITY_SECRET: + break; + case ZT_STATE_OBJECT_PEER_IDENTITY: + break; + case ZT_STATE_OBJECT_NETWORK_CONFIG: + break; + case ZT_STATE_OBJECT_PLANET: + break; + case ZT_STATE_OBJECT_MOON: + break; + default: + return -1; + } + if (f) { + } + return -1; } inline int nodeWirePacketSendFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) @@ -2178,28 +2508,41 @@ public: const uint64_t now = OSUtils::now(); if (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER)) { if (_tcpFallbackTunnel) { - Mutex::Lock _l(_tcpFallbackTunnel->writeBuf_m); - if (!_tcpFallbackTunnel->writeBuf.length()) + Mutex::Lock _l(_tcpFallbackTunnel->writeq_m); + if (_tcpFallbackTunnel->writeq.length() == 0) _phy.setNotifyWritable(_tcpFallbackTunnel->sock,true); - unsigned long mlen = len + 7; - _tcpFallbackTunnel->writeBuf.push_back((char)0x17); - _tcpFallbackTunnel->writeBuf.push_back((char)0x03); - _tcpFallbackTunnel->writeBuf.push_back((char)0x03); // fake TLS 1.2 header - _tcpFallbackTunnel->writeBuf.push_back((char)((mlen >> 8) & 0xff)); - _tcpFallbackTunnel->writeBuf.push_back((char)(mlen & 0xff)); - _tcpFallbackTunnel->writeBuf.push_back((char)4); // IPv4 - _tcpFallbackTunnel->writeBuf.append(reinterpret_cast(reinterpret_cast(&(reinterpret_cast(addr)->sin_addr.s_addr))),4); - _tcpFallbackTunnel->writeBuf.append(reinterpret_cast(reinterpret_cast(&(reinterpret_cast(addr)->sin_port))),2); - _tcpFallbackTunnel->writeBuf.append((const char *)data,len); + const unsigned long mlen = len + 7; + _tcpFallbackTunnel->writeq.push_back((char)0x17); + _tcpFallbackTunnel->writeq.push_back((char)0x03); + _tcpFallbackTunnel->writeq.push_back((char)0x03); // fake TLS 1.2 header + _tcpFallbackTunnel->writeq.push_back((char)((mlen >> 8) & 0xff)); + _tcpFallbackTunnel->writeq.push_back((char)(mlen & 0xff)); + _tcpFallbackTunnel->writeq.push_back((char)4); // IPv4 + _tcpFallbackTunnel->writeq.append(reinterpret_cast(reinterpret_cast(&(reinterpret_cast(addr)->sin_addr.s_addr))),4); + _tcpFallbackTunnel->writeq.append(reinterpret_cast(reinterpret_cast(&(reinterpret_cast(addr)->sin_port))),2); + _tcpFallbackTunnel->writeq.append((const char *)data,len); } else if (((now - _lastSendToGlobalV4) < ZT_TCP_FALLBACK_AFTER)&&((now - _lastSendToGlobalV4) > (ZT_PING_CHECK_INVERVAL / 2))) { bool connected = false; const InetAddress addr(ZT_TCP_FALLBACK_RELAY); - _phy.tcpConnect(reinterpret_cast(&addr),connected); + + TcpConnection *tc = new TcpConnection(); + { + Mutex::Lock _l(_tcpConnections_m); + _tcpConnections.push_back(tc); + } + + tc->type = TcpConnection::TCP_TUNNEL_OUTGOING; + tc->parent = this; + tc->sock = (PhySocket *)0; // set in connect handler + tc->messageSize = 0; + + _phy.tcpConnect(reinterpret_cast(&addr),connected,(void *)tc,true); } } _lastSendToGlobalV4 = now; } #endif // ZT_TCP_FALLBACK_RELAY + } else if (addr->ss_family == AF_INET6) { if (reinterpret_cast(localAddr)->sin6_port != 0) { const uint16_t lp = reinterpret_cast(localAddr)->sin6_port; @@ -2298,39 +2641,22 @@ public: inline void onHttpRequestToServer(TcpConnection *tc) { - char tmpn[256]; + char tmpn[4096]; std::string data; std::string contentType("text/plain"); // default if not changed in handleRequest() unsigned int scode = 404; - bool allow; - { - Mutex::Lock _l(_localConfig_m); - if (_allowManagementFrom.size() == 0) { - allow = (tc->from.ipScope() == InetAddress::IP_SCOPE_LOOPBACK); - } else { - allow = false; - for(std::vector::const_iterator i(_allowManagementFrom.begin());i!=_allowManagementFrom.end();++i) { - if (i->containsAddress(tc->from)) { - allow = true; - break; - } - } - } - } + // Note that we check allowed IP ranges when HTTP connections are first detected in + // phyOnTcpData(). If we made it here the source IP is okay. - if (allow) { - try { - scode = handleControlPlaneHttpRequest(tc->from,tc->parser.method,tc->url,tc->headers,tc->body,data,contentType); - } catch (std::exception &exc) { - fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: %s" ZT_EOL_S,exc.what()); - scode = 500; - } catch ( ... ) { - fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: unknown exceptino" ZT_EOL_S); - scode = 500; - } - } else { - scode = 401; + try { + scode = handleControlPlaneHttpRequest(tc->from,tc->parser.method,tc->url,tc->headers,tc->readq,data,contentType); + } catch (std::exception &exc) { + fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: %s" ZT_EOL_S,exc.what()); + scode = 500; + } catch ( ... ) { + fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: unknown exceptino" ZT_EOL_S); + scode = 500; } const char *scodestr; @@ -2346,19 +2672,16 @@ public: default: scodestr = "Error"; break; } - Utils::snprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\n",scode,scodestr); + Utils::snprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + scode, + scodestr, + contentType.c_str(), + (unsigned long)data.length()); { - Mutex::Lock _l(tc->writeBuf_m); - tc->writeBuf.assign(tmpn); - tc->writeBuf.append("Content-Type: "); - tc->writeBuf.append(contentType); - Utils::snprintf(tmpn,sizeof(tmpn),"\r\nContent-Length: %lu\r\n",(unsigned long)data.length()); - tc->writeBuf.append(tmpn); - if (!tc->shouldKeepAlive) - tc->writeBuf.append("Connection: close\r\n"); - tc->writeBuf.append("\r\n"); + Mutex::Lock _l(tc->writeq_m); + tc->writeq = tmpn; if (tc->parser.method != HTTP_HEAD) - tc->writeBuf.append(data); + tc->writeq.append(data); } _phy.setNotifyWritable(tc->sock,true); @@ -2366,8 +2689,7 @@ public: inline void onHttpResponseFromClient(TcpConnection *tc) { - if (!tc->shouldKeepAlive) - _phy.close(tc->sock); // will call close handler, which deletes from _tcpConnections + _phy.close(tc->sock); } bool shouldBindInterface(const char *ifname,const InetAddress &ifaddr) @@ -2472,10 +2794,10 @@ static int ShttpOnMessageBegin(http_parser *parser) tc->currentHeaderField = ""; tc->currentHeaderValue = ""; tc->messageSize = 0; - tc->url = ""; - tc->status = ""; + tc->url.clear(); + tc->status.clear(); tc->headers.clear(); - tc->body = ""; + tc->readq.clear(); return 0; } static int ShttpOnUrl(http_parser *parser,const char *ptr,size_t length) @@ -2492,16 +2814,7 @@ static int ShttpOnStatus(http_parser *parser,const char *ptr,size_t length) #else static int ShttpOnStatus(http_parser *parser) #endif -{ - /* - TcpConnection *tc = reinterpret_cast(parser->data); - tc->messageSize += (unsigned long)length; - if (tc->messageSize > ZT_MAX_HTTP_MESSAGE_SIZE) - return -1; - tc->status.append(ptr,length); - */ - return 0; -} +{ return 0; } static int ShttpOnHeaderField(http_parser *parser,const char *ptr,size_t length) { TcpConnection *tc = reinterpret_cast(parser->data); @@ -2539,14 +2852,12 @@ static int ShttpOnBody(http_parser *parser,const char *ptr,size_t length) tc->messageSize += (unsigned long)length; if (tc->messageSize > ZT_MAX_HTTP_MESSAGE_SIZE) return -1; - tc->body.append(ptr,length); + tc->readq.append(ptr,length); return 0; } static int ShttpOnMessageComplete(http_parser *parser) { TcpConnection *tc = reinterpret_cast(parser->data); - tc->shouldKeepAlive = (http_should_keep_alive(parser) != 0); - tc->lastActivity = OSUtils::now(); if (tc->type == TcpConnection::TCP_HTTP_INCOMING) { tc->parent->onHttpRequestToServer(tc); } else { -- cgit v1.2.3 From dd68c207f4ed13c4f71bc4d019732f21d633575c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Jun 2017 08:51:49 -0700 Subject: Stuff old Cluster code in attic. --- Cluster.cpp | 1042 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cluster.hpp | 463 ++++++++++++++++++++++++ node/Cluster.cpp | 1042 ------------------------------------------------------ node/Cluster.hpp | 463 ------------------------ objects.mk | 1 - 5 files changed, 1505 insertions(+), 1506 deletions(-) create mode 100644 Cluster.cpp create mode 100644 Cluster.hpp delete mode 100644 node/Cluster.cpp delete mode 100644 node/Cluster.hpp (limited to 'node') diff --git a/Cluster.cpp b/Cluster.cpp new file mode 100644 index 00000000..119aec29 --- /dev/null +++ b/Cluster.cpp @@ -0,0 +1,1042 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../version.h" + +#include "Cluster.hpp" +#include "RuntimeEnvironment.hpp" +#include "MulticastGroup.hpp" +#include "CertificateOfMembership.hpp" +#include "Salsa20.hpp" +#include "Poly1305.hpp" +#include "Identity.hpp" +#include "Topology.hpp" +#include "Packet.hpp" +#include "Switch.hpp" +#include "Node.hpp" +#include "Network.hpp" +#include "Array.hpp" + +namespace ZeroTier { + +static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2) + throw() +{ + double dx = ((double)x2 - (double)x1); + double dy = ((double)y2 - (double)y1); + double dz = ((double)z2 - (double)z1); + return sqrt((dx * dx) + (dy * dy) + (dz * dz)); +} + +// An entry in _ClusterSendQueue +struct _ClusterSendQueueEntry +{ + uint64_t timestamp; + Address fromPeerAddress; + Address toPeerAddress; + // if we ever support larger transport MTUs this must be increased + unsigned char data[ZT_CLUSTER_SEND_QUEUE_DATA_MAX]; + unsigned int len; + bool unite; +}; + +// A multi-index map with entry memory pooling -- this allows our queue to +// be O(log(N)) and is complex enough that it makes the code a lot cleaner +// to break it out from Cluster. +class _ClusterSendQueue +{ +public: + _ClusterSendQueue() : + _poolCount(0) {} + ~_ClusterSendQueue() {} // memory is automatically freed when _chunks is destroyed + + inline void enqueue(uint64_t now,const Address &from,const Address &to,const void *data,unsigned int len,bool unite) + { + if (len > ZT_CLUSTER_SEND_QUEUE_DATA_MAX) + return; + + Mutex::Lock _l(_lock); + + // Delete oldest queue entry for this sender if this enqueue() would take them over the per-sender limit + { + std::set< std::pair >::iterator qi(_bySrc.lower_bound(std::pair(from,(_ClusterSendQueueEntry *)0))); + std::set< std::pair >::iterator oldest(qi); + unsigned long countForSender = 0; + while ((qi != _bySrc.end())&&(qi->first == from)) { + if (qi->second->timestamp < oldest->second->timestamp) + oldest = qi; + ++countForSender; + ++qi; + } + if (countForSender >= ZT_CLUSTER_MAX_QUEUE_PER_SENDER) { + _byDest.erase(std::pair(oldest->second->toPeerAddress,oldest->second)); + _pool[_poolCount++] = oldest->second; + _bySrc.erase(oldest); + } + } + + _ClusterSendQueueEntry *e; + if (_poolCount > 0) { + e = _pool[--_poolCount]; + } else { + if (_chunks.size() >= ZT_CLUSTER_MAX_QUEUE_CHUNKS) + return; // queue is totally full! + _chunks.push_back(Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE>()); + e = &(_chunks.back().data[0]); + for(unsigned int i=1;itimestamp = now; + e->fromPeerAddress = from; + e->toPeerAddress = to; + memcpy(e->data,data,len); + e->len = len; + e->unite = unite; + + _bySrc.insert(std::pair(from,e)); + _byDest.insert(std::pair(to,e)); + } + + inline void expire(uint64_t now) + { + Mutex::Lock _l(_lock); + for(std::set< std::pair >::iterator qi(_bySrc.begin());qi!=_bySrc.end();) { + if ((now - qi->second->timestamp) > ZT_CLUSTER_QUEUE_EXPIRATION) { + _byDest.erase(std::pair(qi->second->toPeerAddress,qi->second)); + _pool[_poolCount++] = qi->second; + _bySrc.erase(qi++); + } else ++qi; + } + } + + /** + * Get and dequeue entries for a given destination address + * + * After use these entries must be returned with returnToPool()! + * + * @param dest Destination address + * @param results Array to fill with results + * @param maxResults Size of results[] in pointers + * @return Number of actual results returned + */ + inline unsigned int getByDest(const Address &dest,_ClusterSendQueueEntry **results,unsigned int maxResults) + { + unsigned int count = 0; + Mutex::Lock _l(_lock); + std::set< std::pair >::iterator qi(_byDest.lower_bound(std::pair(dest,(_ClusterSendQueueEntry *)0))); + while ((qi != _byDest.end())&&(qi->first == dest)) { + _bySrc.erase(std::pair(qi->second->fromPeerAddress,qi->second)); + results[count++] = qi->second; + if (count == maxResults) + break; + _byDest.erase(qi++); + } + return count; + } + + /** + * Return entries to pool after use + * + * @param entries Array of entries + * @param count Number of entries + */ + inline void returnToPool(_ClusterSendQueueEntry **entries,unsigned int count) + { + Mutex::Lock _l(_lock); + for(unsigned int i=0;i > _chunks; + _ClusterSendQueueEntry *_pool[ZT_CLUSTER_QUEUE_CHUNK_SIZE * ZT_CLUSTER_MAX_QUEUE_CHUNKS]; + unsigned long _poolCount; + std::set< std::pair > _bySrc; + std::set< std::pair > _byDest; + Mutex _lock; +}; + +Cluster::Cluster( + const RuntimeEnvironment *renv, + uint16_t id, + const std::vector &zeroTierPhysicalEndpoints, + int32_t x, + int32_t y, + int32_t z, + void (*sendFunction)(void *,unsigned int,const void *,unsigned int), + void *sendFunctionArg, + int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), + void *addressToLocationFunctionArg) : + RR(renv), + _sendQueue(new _ClusterSendQueue()), + _sendFunction(sendFunction), + _sendFunctionArg(sendFunctionArg), + _addressToLocationFunction(addressToLocationFunction), + _addressToLocationFunctionArg(addressToLocationFunctionArg), + _x(x), + _y(y), + _z(z), + _id(id), + _zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints), + _members(new _Member[ZT_CLUSTER_MAX_MEMBERS]), + _lastFlushed(0), + _lastCleanedRemotePeers(0), + _lastCleanedQueue(0) +{ + uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + + // Generate master secret by hashing the secret from our Identity key pair + RR->identity.sha512PrivateKey(_masterSecret); + + // Generate our inbound message key, which is the master secret XORed with our ID and hashed twice + memcpy(stmp,_masterSecret,sizeof(stmp)); + stmp[0] ^= Utils::hton(id); + SHA512::hash(stmp,stmp,sizeof(stmp)); + SHA512::hash(stmp,stmp,sizeof(stmp)); + memcpy(_key,stmp,sizeof(_key)); + Utils::burn(stmp,sizeof(stmp)); +} + +Cluster::~Cluster() +{ + Utils::burn(_masterSecret,sizeof(_masterSecret)); + Utils::burn(_key,sizeof(_key)); + delete [] _members; + delete _sendQueue; +} + +void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) +{ + Buffer dmsg; + { + // FORMAT: <[16] iv><[8] MAC><... data> + if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH)) + return; + + // 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV + char keytmp[32]; + memcpy(keytmp,_key,32); + for(int i=0;i<8;++i) + keytmp[i] ^= reinterpret_cast(msg)[i]; + Salsa20 s20(keytmp,reinterpret_cast(msg) + 8); + Utils::burn(keytmp,sizeof(keytmp)); + + // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") + char polykey[ZT_POLY1305_KEY_LEN]; + memset(polykey,0,sizeof(polykey)); + s20.crypt12(polykey,polykey,sizeof(polykey)); + + // Compute 16-byte MAC + char mac[ZT_POLY1305_MAC_LEN]; + Poly1305::compute(mac,reinterpret_cast(msg) + 24,len - 24,polykey); + + // Check first 8 bytes of MAC against 64-bit MAC in stream + if (!Utils::secureEq(mac,reinterpret_cast(msg) + 16,8)) + return; + + // Decrypt! + dmsg.setSize(len - 24); + s20.crypt12(reinterpret_cast(msg) + 24,const_cast(dmsg.data()),dmsg.size()); + } + + if (dmsg.size() < 4) + return; + const uint16_t fromMemberId = dmsg.at(0); + unsigned int ptr = 2; + if (fromMemberId == _id) // sanity check: we don't talk to ourselves + return; + const uint16_t toMemberId = dmsg.at(ptr); + ptr += 2; + if (toMemberId != _id) // sanity check: message not for us? + return; + + { // make sure sender is actually considered a member + Mutex::Lock _l3(_memberIds_m); + if (std::find(_memberIds.begin(),_memberIds.end(),fromMemberId) == _memberIds.end()) + return; + } + + try { + while (ptr < dmsg.size()) { + const unsigned int mlen = dmsg.at(ptr); ptr += 2; + const unsigned int nextPtr = ptr + mlen; + if (nextPtr > dmsg.size()) + break; + + int mtype = -1; + try { + switch((StateMessageType)(mtype = (int)dmsg[ptr++])) { + default: + break; + + case CLUSTER_MESSAGE_ALIVE: { + _Member &m = _members[fromMemberId]; + Mutex::Lock mlck(m.lock); + ptr += 7; // skip version stuff, not used yet + m.x = dmsg.at(ptr); ptr += 4; + m.y = dmsg.at(ptr); ptr += 4; + m.z = dmsg.at(ptr); ptr += 4; + ptr += 8; // skip local clock, not used + m.load = dmsg.at(ptr); ptr += 8; + m.peers = dmsg.at(ptr); ptr += 8; + ptr += 8; // skip flags, unused +#ifdef ZT_TRACE + std::string addrs; +#endif + unsigned int physicalAddressCount = dmsg[ptr++]; + m.zeroTierPhysicalEndpoints.clear(); + for(unsigned int i=0;i 0) + addrs.push_back(','); + addrs.append(m.zeroTierPhysicalEndpoints.back().toString()); + } +#endif + } +#ifdef ZT_TRACE + if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) { + TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str()); + } +#endif + m.lastReceivedAliveAnnouncement = RR->node->now(); + } break; + + case CLUSTER_MESSAGE_HAVE_PEER: { + Identity id; + ptr += id.deserialize(dmsg,ptr); + if (id) { + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)]; + if (!rp.lastHavePeerReceived) { + RR->topology->saveIdentity((void *)0,id); + RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH); + } + rp.lastHavePeerReceived = RR->node->now(); + } + + _ClusterSendQueueEntry *q[16384]; // 16384 is "tons" + unsigned int qc = _sendQueue->getByDest(id.address(),q,16384); + for(unsigned int i=0;irelayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite); + _sendQueue->returnToPool(q,qc); + + TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc); + } + } break; + + case CLUSTER_MESSAGE_WANT_PEER: { + const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + SharedPtr peer(RR->topology->getPeerNoCache(zeroTierAddress)); + if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) { + Buffer<1024> buf; + peer->identity().serialize(buf); + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); + } + } break; + + case CLUSTER_MESSAGE_REMOTE_PACKET: { + const unsigned int plen = dmsg.at(ptr); ptr += 2; + if (plen) { + Packet remotep(dmsg.field(ptr,plen),plen); ptr += plen; + //TRACE("remote %s from %s via %u (%u bytes)",Packet::verbString(remotep.verb()),remotep.source().toString().c_str(),fromMemberId,plen); + switch(remotep.verb()) { + case Packet::VERB_WHOIS: _doREMOTE_WHOIS(fromMemberId,remotep); break; + case Packet::VERB_MULTICAST_GATHER: _doREMOTE_MULTICAST_GATHER(fromMemberId,remotep); break; + default: break; // ignore things we don't care about across cluster + } + } + } break; + + case CLUSTER_MESSAGE_PROXY_UNITE: { + const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const unsigned int numRemotePeerPaths = dmsg[ptr++]; + InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max + for(unsigned int i=0;inode->now(); + SharedPtr localPeer(RR->topology->getPeerNoCache(localPeerAddress)); + if ((localPeer)&&(numRemotePeerPaths > 0)) { + InetAddress bestLocalV4,bestLocalV6; + localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6); + + InetAddress bestRemoteV4,bestRemoteV6; + for(unsigned int i=0;iidentity.address(),Packet::VERB_RENDEZVOUS); + rendezvousForLocal.append((uint8_t)0); + remotePeerAddress.appendTo(rendezvousForLocal); + + Buffer<2048> rendezvousForRemote; + remotePeerAddress.appendTo(rendezvousForRemote); + rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS); + rendezvousForRemote.addSize(2); // space for actual packet payload length + rendezvousForRemote.append((uint8_t)0); // flags == 0 + localPeerAddress.appendTo(rendezvousForRemote); + + bool haveMatch = false; + if ((bestLocalV6)&&(bestRemoteV6)) { + haveMatch = true; + + rendezvousForLocal.append((uint16_t)bestRemoteV6.port()); + rendezvousForLocal.append((uint8_t)16); + rendezvousForLocal.append(bestRemoteV6.rawIpData(),16); + + rendezvousForRemote.append((uint16_t)bestLocalV6.port()); + rendezvousForRemote.append((uint8_t)16); + rendezvousForRemote.append(bestLocalV6.rawIpData(),16); + rendezvousForRemote.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 16)); + } else if ((bestLocalV4)&&(bestRemoteV4)) { + haveMatch = true; + + rendezvousForLocal.append((uint16_t)bestRemoteV4.port()); + rendezvousForLocal.append((uint8_t)4); + rendezvousForLocal.append(bestRemoteV4.rawIpData(),4); + + rendezvousForRemote.append((uint16_t)bestLocalV4.port()); + rendezvousForRemote.append((uint8_t)4); + rendezvousForRemote.append(bestLocalV4.rawIpData(),4); + rendezvousForRemote.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 4)); + } + + if (haveMatch) { + { + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size()); + } + RR->sw->send((void *)0,rendezvousForLocal,true); + } + } + } break; + + case CLUSTER_MESSAGE_PROXY_SEND: { + const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; + const Packet::Verb verb = (Packet::Verb)dmsg[ptr++]; + const unsigned int len = dmsg.at(ptr); ptr += 2; + Packet outp(rcpt,RR->identity.address(),verb); + outp.append(dmsg.field(ptr,len),len); ptr += len; + RR->sw->send((void *)0,outp,true); + //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); + } break; + + case CLUSTER_MESSAGE_NETWORK_CONFIG: { + const SharedPtr network(RR->node->network(dmsg.at(ptr))); + if (network) { + // Copy into a Packet just to conform to Network API. Eventually + // will want to refactor. + network->handleConfigChunk((void *)0,0,Address(),Buffer(dmsg),ptr); + } + } break; + } + } catch ( ... ) { + TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype); + // drop invalids + } + + ptr = nextPtr; + } + } catch ( ... ) { + TRACE("invalid message (outer loop), discarding"); + // drop invalids + } +} + +void Cluster::broadcastHavePeer(const Identity &id) +{ + Buffer<1024> buf; + id.serialize(buf); + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); + } +} + +void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) +{ + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); + } +} + +int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) +{ + const uint64_t now = RR->node->now(); + mostRecentTs = 0; + int mostRecentMemberId = -1; + { + Mutex::Lock _l2(_remotePeers_m); + std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); + for(;;) { + if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) + break; + else if (rpe->second.lastHavePeerReceived > mostRecentTs) { + mostRecentTs = rpe->second.lastHavePeerReceived; + memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); + mostRecentMemberId = (int)rpe->first.second; + } + ++rpe; + } + } + + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) + mostRecentMemberId = -1; + + bool sendWantPeer = true; + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER + } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + } + } + } + } + + return mostRecentMemberId; +} + +bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len) +{ + if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check + return false; + Mutex::Lock _l2(_members[mostRecentMemberId].lock); + for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { + for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { + if (i1->ss_family == i2->ss_family) { + TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); + RR->node->putPacket((void *)0,*i1,*i2,data,len); + return true; + } + } + } + return false; +} + +void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) +{ + if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check + return; + + const uint64_t now = RR->node->now(); + + uint64_t mostRecentTs = 0; + int mostRecentMemberId = -1; + { + Mutex::Lock _l2(_remotePeers_m); + std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); + for(;;) { + if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) + break; + else if (rpe->second.lastHavePeerReceived > mostRecentTs) { + mostRecentTs = rpe->second.lastHavePeerReceived; + mostRecentMemberId = (int)rpe->first.second; + } + ++rpe; + } + } + + const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; + if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { + // Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing + const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0)); + + // Poll everyone with WANT_PEER if the age of our most recent entry is + // approaching expiration (or has expired, or does not exist). + bool sendWantPeer = true; + { + Mutex::Lock _l(_remotePeers_m); + _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; + if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { + rp.lastSentWantPeer = now; + } else { + sendWantPeer = false; // don't flood WANT_PEER + } + } + if (sendWantPeer) { + char tmp[ZT_ADDRESS_LENGTH]; + toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); + } + } + } + + // If there isn't a good place to send via, then enqueue this for retrying + // later and return after having broadcasted a WANT_PEER. + if (enqueueAndWait) { + TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str()); + _sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite); + return; + } + } + + if (mostRecentMemberId >= 0) { + Buffer<1024> buf; + if (unite) { + InetAddress v4,v6; + if (fromPeerAddress) { + SharedPtr fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); + if (fromPeer) + fromPeer->getRendezvousAddresses(now,v4,v6); + } + uint8_t addrCount = 0; + if (v4) + ++addrCount; + if (v6) + ++addrCount; + if (addrCount) { + toPeerAddress.appendTo(buf); + fromPeerAddress.appendTo(buf); + buf.append(addrCount); + if (v4) + v4.serialize(buf); + if (v6) + v6.serialize(buf); + } + } + + { + Mutex::Lock _l2(_members[mostRecentMemberId].lock); + if (buf.size() > 0) + _send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size()); + + for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { + for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { + if (i1->ss_family == i2->ss_family) { + TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); + RR->node->putPacket((void *)0,*i1,*i2,data,len); + return; + } + } + } + + TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId); + } + } +} + +void Cluster::sendDistributedQuery(const Packet &pkt) +{ + Buffer<4096> buf; + buf.append((uint16_t)pkt.size()); + buf.append(pkt.data(),pkt.size()); + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + _send(*mid,CLUSTER_MESSAGE_REMOTE_PACKET,buf.data(),buf.size()); + } +} + +void Cluster::doPeriodicTasks() +{ + const uint64_t now = RR->node->now(); + + if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) { + _lastFlushed = now; + + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + + if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) { + _members[*mid].lastAnnouncedAliveTo = now; + + Buffer<2048> alive; + alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR); + alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR); + alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); + alive.append((uint8_t)ZT_PROTO_VERSION); + if (_addressToLocationFunction) { + alive.append((int32_t)_x); + alive.append((int32_t)_y); + alive.append((int32_t)_z); + } else { + alive.append((int32_t)0); + alive.append((int32_t)0); + alive.append((int32_t)0); + } + alive.append((uint64_t)now); + alive.append((uint64_t)0); // TODO: compute and send load average + alive.append((uint64_t)RR->topology->countActive(now)); + alive.append((uint64_t)0); // unused/reserved flags + alive.append((uint8_t)_zeroTierPhysicalEndpoints.size()); + for(std::vector::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe) + pe->serialize(alive); + _send(*mid,CLUSTER_MESSAGE_ALIVE,alive.data(),alive.size()); + } + + _flush(*mid); + } + } + + if ((now - _lastCleanedRemotePeers) >= (ZT_PEER_ACTIVITY_TIMEOUT * 2)) { + _lastCleanedRemotePeers = now; + + Mutex::Lock _l(_remotePeers_m); + for(std::map< std::pair,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) { + if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT) + _remotePeers.erase(rp++); + else ++rp; + } + } + + if ((now - _lastCleanedQueue) >= ZT_CLUSTER_QUEUE_EXPIRATION) { + _lastCleanedQueue = now; + _sendQueue->expire(now); + } +} + +void Cluster::addMember(uint16_t memberId) +{ + if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id)) + return; + + Mutex::Lock _l2(_members[memberId].lock); + + { + Mutex::Lock _l(_memberIds_m); + if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end()) + return; + _memberIds.push_back(memberId); + std::sort(_memberIds.begin(),_memberIds.end()); + } + + _members[memberId].clear(); + + // Generate this member's message key from the master and its ID + uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + memcpy(stmp,_masterSecret,sizeof(stmp)); + stmp[0] ^= Utils::hton(memberId); + SHA512::hash(stmp,stmp,sizeof(stmp)); + SHA512::hash(stmp,stmp,sizeof(stmp)); + memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key)); + Utils::burn(stmp,sizeof(stmp)); + + // Prepare q + _members[memberId].q.clear(); + char iv[16]; + Utils::getSecureRandom(iv,16); + _members[memberId].q.append(iv,16); + _members[memberId].q.addSize(8); // room for MAC + _members[memberId].q.append((uint16_t)_id); + _members[memberId].q.append((uint16_t)memberId); +} + +void Cluster::removeMember(uint16_t memberId) +{ + Mutex::Lock _l(_memberIds_m); + std::vector newMemberIds; + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + if (*mid != memberId) + newMemberIds.push_back(*mid); + } + _memberIds = newMemberIds; +} + +bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload) +{ + if (_addressToLocationFunction) { + // Pick based on location if it can be determined + int px = 0,py = 0,pz = 0; + if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast(&peerPhysicalAddress),&px,&py,&pz) == 0) { + TRACE("no geolocation data for %s",peerPhysicalAddress.toIpString().c_str()); + return false; + } + + // Find member closest to this peer + const uint64_t now = RR->node->now(); + std::vector best; + const double currentDistance = _dist3d(_x,_y,_z,px,py,pz); + double bestDistance = (offload ? 2147483648.0 : currentDistance); +#ifdef ZT_TRACE + unsigned int bestMember = _id; +#endif + { + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + _Member &m = _members[*mid]; + Mutex::Lock _ml(m.lock); + + // Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to + if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) { + const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz); + if (mdist < bestDistance) { + bestDistance = mdist; +#ifdef ZT_TRACE + bestMember = *mid; +#endif + best = m.zeroTierPhysicalEndpoints; + } + } + } + } + + // Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family + for(std::vector::const_iterator a(best.begin());a!=best.end();++a) { + if (a->ss_family == peerPhysicalAddress.ss_family) { + TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str()); + redirectTo = *a; + return true; + } + } + TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance); + return false; + } else { + // TODO: pick based on load if no location info? + return false; + } +} + +bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const +{ + Mutex::Lock _l(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + Mutex::Lock _l2(_members[*mid].lock); + for(std::vector::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) { + if (ip == *i2) + return true; + } + } + return false; +} + +void Cluster::status(ZT_ClusterStatus &status) const +{ + const uint64_t now = RR->node->now(); + memset(&status,0,sizeof(ZT_ClusterStatus)); + + status.myId = _id; + + { + ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); + s->id = _id; + s->alive = 1; + s->x = _x; + s->y = _y; + s->z = _z; + s->load = 0; // TODO + s->peers = RR->topology->countActive(now); + for(std::vector::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) { + if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check + break; + memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); + } + } + + { + Mutex::Lock _l1(_memberIds_m); + for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { + if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check + break; + + _Member &m = _members[*mid]; + Mutex::Lock ml(m.lock); + + ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); + s->id = *mid; + s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement)); + s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0; + s->x = m.x; + s->y = m.y; + s->z = m.z; + s->load = m.load; + s->peers = m.peers; + for(std::vector::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) { + if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check + break; + memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); + } + } + } +} + +void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len) +{ + if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check + return; + _Member &m = _members[memberId]; + // assumes m.lock is locked! + if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH) + _flush(memberId); + m.q.append((uint16_t)(len + 1)); + m.q.append((uint8_t)type); + m.q.append(msg,len); +} + +void Cluster::_flush(uint16_t memberId) +{ + _Member &m = _members[memberId]; + // assumes m.lock is locked! + if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID + // Create key from member's key and IV + char keytmp[32]; + memcpy(keytmp,m.key,32); + for(int i=0;i<8;++i) + keytmp[i] ^= m.q[i]; + Salsa20 s20(keytmp,m.q.field(8,8)); + Utils::burn(keytmp,sizeof(keytmp)); + + // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") + char polykey[ZT_POLY1305_KEY_LEN]; + memset(polykey,0,sizeof(polykey)); + s20.crypt12(polykey,polykey,sizeof(polykey)); + + // Encrypt m.q in place + s20.crypt12(reinterpret_cast(m.q.data()) + 24,const_cast(reinterpret_cast(m.q.data())) + 24,m.q.size() - 24); + + // Add MAC for authentication (encrypt-then-MAC) + char mac[ZT_POLY1305_MAC_LEN]; + Poly1305::compute(mac,reinterpret_cast(m.q.data()) + 24,m.q.size() - 24,polykey); + memcpy(m.q.field(16,8),mac,8); + + // Send! + _sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size()); + + // Prepare for more + m.q.clear(); + char iv[16]; + Utils::getSecureRandom(iv,16); + m.q.append(iv,16); + m.q.addSize(8); // room for MAC + m.q.append((uint16_t)_id); // from member ID + m.q.append((uint16_t)memberId); // to member ID + } +} + +void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep) +{ + if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) { + Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH))); + if (queried) { + Buffer<1024> routp; + remotep.source().appendTo(routp); + routp.append((uint8_t)Packet::VERB_OK); + routp.addSize(2); // space for length + routp.append((uint8_t)Packet::VERB_WHOIS); + routp.append(remotep.packetId()); + queried.serialize(routp); + routp.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); + + TRACE("responding to remote WHOIS from %s @ %u with identity of %s",remotep.source().toString().c_str(),(unsigned int)fromMemberId,queried.address().toString().c_str()); + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); + } + } +} + +void Cluster::_doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep) +{ + const uint64_t nwid = remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); + const MulticastGroup mg(MAC(remotep.field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); + unsigned int gatherLimit = remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); + const Address remotePeerAddress(remotep.source()); + + if (gatherLimit) { + Buffer routp; + remotePeerAddress.appendTo(routp); + routp.append((uint8_t)Packet::VERB_OK); + routp.addSize(2); // space for length + routp.append((uint8_t)Packet::VERB_MULTICAST_GATHER); + routp.append(remotep.packetId()); + routp.append(nwid); + mg.mac().appendTo(routp); + routp.append((uint32_t)mg.adi()); + + if (gatherLimit > ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5)) + gatherLimit = ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5); + if (RR->mc->gather(remotePeerAddress,nwid,mg,routp,gatherLimit)) { + routp.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); + + TRACE("responding to remote MULTICAST_GATHER from %s @ %u with %u bytes",remotePeerAddress.toString().c_str(),(unsigned int)fromMemberId,routp.size()); + Mutex::Lock _l2(_members[fromMemberId].lock); + _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); + } + } +} + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER diff --git a/Cluster.hpp b/Cluster.hpp new file mode 100644 index 00000000..74b091f5 --- /dev/null +++ b/Cluster.hpp @@ -0,0 +1,463 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CLUSTER_HPP +#define ZT_CLUSTER_HPP + +#ifdef ZT_ENABLE_CLUSTER + +#include + +#include "Constants.hpp" +#include "../include/ZeroTierOne.h" +#include "Address.hpp" +#include "InetAddress.hpp" +#include "SHA512.hpp" +#include "Utils.hpp" +#include "Buffer.hpp" +#include "Mutex.hpp" +#include "SharedPtr.hpp" +#include "Hashtable.hpp" +#include "Packet.hpp" +#include "SharedPtr.hpp" + +/** + * Timeout for cluster members being considered "alive" + * + * A cluster member is considered dead and will no longer have peers + * redirected to it if we have not heard a heartbeat in this long. + */ +#define ZT_CLUSTER_TIMEOUT 5000 + +/** + * Desired period between doPeriodicTasks() in milliseconds + */ +#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 20 + +/** + * How often to flush outgoing message queues (maximum interval) + */ +#define ZT_CLUSTER_FLUSH_PERIOD ZT_CLUSTER_PERIODIC_TASK_PERIOD + +/** + * Maximum number of queued outgoing packets per sender address + */ +#define ZT_CLUSTER_MAX_QUEUE_PER_SENDER 16 + +/** + * Expiration time for send queue entries + */ +#define ZT_CLUSTER_QUEUE_EXPIRATION 3000 + +/** + * Chunk size for allocating queue entries + * + * Queue entries are allocated in chunks of this many and are added to a pool. + * ZT_CLUSTER_MAX_QUEUE_GLOBAL must be evenly divisible by this. + */ +#define ZT_CLUSTER_QUEUE_CHUNK_SIZE 32 + +/** + * Maximum number of chunks to ever allocate + * + * This is a global sanity limit to prevent resource exhaustion attacks. It + * works out to about 600mb of RAM. You'll never see this on a normal edge + * node. We're unlikely to see this on a root server unless someone is DOSing + * us. In that case cluster relaying will be affected but other functions + * should continue to operate normally. + */ +#define ZT_CLUSTER_MAX_QUEUE_CHUNKS 8194 + +/** + * Max data per queue entry + */ +#define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500 + +/** + * We won't send WANT_PEER to other members more than every (ms) per recipient + */ +#define ZT_CLUSTER_WANT_PEER_EVERY 1000 + +namespace ZeroTier { + +class RuntimeEnvironment; +class MulticastGroup; +class Peer; +class Identity; + +// Internal class implemented inside Cluster.cpp +class _ClusterSendQueue; + +/** + * Multi-homing cluster state replication and packet relaying + * + * Multi-homing means more than one node sharing the same ZeroTier identity. + * There is nothing in the protocol to prevent this, but to make it work well + * requires the devices sharing an identity to cooperate and share some + * information. + * + * There are three use cases we want to fulfill: + * + * (1) Multi-homing of root servers with handoff for efficient routing, + * HA, and load balancing across many commodity nodes. + * (2) Multi-homing of network controllers for the same reason. + * (3) Multi-homing of nodes on virtual networks, such as domain servers + * and other important endpoints. + * + * These use cases are in order of escalating difficulty. The initial + * version of Cluster is aimed at satisfying the first, though you are + * free to try #2 and #3. + */ +class Cluster +{ +public: + /** + * State message types + */ + enum StateMessageType + { + CLUSTER_MESSAGE_NOP = 0, + + /** + * This cluster member is alive: + * <[2] version minor> + * <[2] version major> + * <[2] version revision> + * <[1] protocol version> + * <[4] X location (signed 32-bit)> + * <[4] Y location (signed 32-bit)> + * <[4] Z location (signed 32-bit)> + * <[8] local clock at this member> + * <[8] load average> + * <[8] number of peers> + * <[8] flags (currently unused, must be zero)> + * <[1] number of preferred ZeroTier endpoints> + * <[...] InetAddress(es) of preferred ZeroTier endpoint(s)> + * + * Cluster members constantly broadcast an alive heartbeat and will only + * receive peer redirects if they've done so within the timeout. + */ + CLUSTER_MESSAGE_ALIVE = 1, + + /** + * Cluster member has this peer: + * <[...] serialized identity of peer> + * + * This is typically sent in response to WANT_PEER but can also be pushed + * to prepopulate if this makes sense. + */ + CLUSTER_MESSAGE_HAVE_PEER = 2, + + /** + * Cluster member wants this peer: + * <[5] ZeroTier address of peer> + * + * Members that have a direct link to this peer will respond with + * HAVE_PEER. + */ + CLUSTER_MESSAGE_WANT_PEER = 3, + + /** + * A remote packet that we should also possibly respond to: + * <[2] 16-bit length of remote packet> + * <[...] remote packet payload> + * + * Cluster members may relay requests by relaying the request packet. + * These may include requests such as WHOIS and MULTICAST_GATHER. The + * packet must be already decrypted, decompressed, and authenticated. + * + * This can only be used for small request packets as per the cluster + * message size limit, but since these are the only ones in question + * this is fine. + * + * If a response is generated it is sent via PROXY_SEND. + */ + CLUSTER_MESSAGE_REMOTE_PACKET = 4, + + /** + * Request that VERB_RENDEZVOUS be sent to a peer that we have: + * <[5] ZeroTier address of peer on recipient's side> + * <[5] ZeroTier address of peer on sender's side> + * <[1] 8-bit number of sender's peer's active path addresses> + * <[...] series of serialized InetAddresses of sender's peer's paths> + * + * This requests that we perform NAT-t introduction between a peer that + * we have and one on the sender's side. The sender furnishes contact + * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours + * directly and with PROXY_SEND to theirs. + */ + CLUSTER_MESSAGE_PROXY_UNITE = 5, + + /** + * Request that a cluster member send a packet to a locally-known peer: + * <[5] ZeroTier address of recipient> + * <[1] packet verb> + * <[2] length of packet payload> + * <[...] packet payload> + * + * This differs from RELAY in that it requests the receiving cluster + * member to actually compose a ZeroTier Packet from itself to the + * provided recipient. RELAY simply says "please forward this blob." + * RELAY is used to implement peer-to-peer relaying with RENDEZVOUS, + * while PROXY_SEND is used to implement proxy sending (which right + * now is only used to send RENDEZVOUS). + */ + CLUSTER_MESSAGE_PROXY_SEND = 6, + + /** + * Replicate a network config for a network we belong to: + * <[...] network config chunk> + * + * This is used by clusters to avoid every member having to query + * for the same netconf for networks all members belong to. + * + * The first field of a network config chunk is the network ID, + * so this can be checked to look up the network on receipt. + */ + CLUSTER_MESSAGE_NETWORK_CONFIG = 7 + }; + + /** + * Construct a new cluster + */ + Cluster( + const RuntimeEnvironment *renv, + uint16_t id, + const std::vector &zeroTierPhysicalEndpoints, + int32_t x, + int32_t y, + int32_t z, + void (*sendFunction)(void *,unsigned int,const void *,unsigned int), + void *sendFunctionArg, + int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), + void *addressToLocationFunctionArg); + + ~Cluster(); + + /** + * @return This cluster member's ID + */ + inline uint16_t id() const throw() { return _id; } + + /** + * Handle an incoming intra-cluster message + * + * @param data Message data + * @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH) + */ + void handleIncomingStateMessage(const void *msg,unsigned int len); + + /** + * Broadcast that we have a given peer + * + * This should be done when new peers are first contacted. + * + * @param id Identity of peer + */ + void broadcastHavePeer(const Identity &id); + + /** + * Broadcast a network config chunk to other members of cluster + * + * @param chunk Chunk data + * @param len Length of chunk + */ + void broadcastNetworkConfigChunk(const void *chunk,unsigned int len); + + /** + * If the cluster has this peer, prepare the packet to send via cluster + * + * Note that outp is only armored (or modified at all) if the return value is a member ID. + * + * @param toPeerAddress Value of outp.destination(), simply to save additional lookup + * @param ts Result: set to time of last HAVE_PEER from the cluster + * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes + * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() + */ + int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); + + /** + * Send data via cluster front plane (packet head or fragment) + * + * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster() + * @param toPeerAddress Destination peer address + * @param data Packet or packet fragment data + * @param len Length of packet or fragment + * @return True if packet was sent (and outp was modified via armoring) + */ + bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len); + + /** + * Relay a packet via the cluster + * + * This is used in the outgoing packet and relaying logic in Switch to + * relay packets to other cluster members. It isn't PROXY_SEND-- that is + * used internally in Cluster to send responses to peer queries. + * + * @param fromPeerAddress Source peer address (if known, should be NULL for fragments) + * @param toPeerAddress Destination peer address + * @param data Packet or packet fragment data + * @param len Length of packet or fragment + * @param unite If true, also request proxy unite across cluster + */ + void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite); + + /** + * Send a distributed query to other cluster members + * + * Some queries such as WHOIS or MULTICAST_GATHER need a response from other + * cluster members. Replies (if any) will be sent back to the peer via + * PROXY_SEND across the cluster. + * + * @param pkt Packet to distribute + */ + void sendDistributedQuery(const Packet &pkt); + + /** + * Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds. + */ + void doPeriodicTasks(); + + /** + * Add a member ID to this cluster + * + * @param memberId Member ID + */ + void addMember(uint16_t memberId); + + /** + * Remove a member ID from this cluster + * + * @param memberId Member ID to remove + */ + void removeMember(uint16_t memberId); + + /** + * Find a better cluster endpoint for this peer (if any) + * + * @param redirectTo InetAddress to be set to a better endpoint (if there is one) + * @param peerAddress Address of peer to (possibly) redirect + * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address()) + * @param offload Always redirect if possible -- can be used to offload peers during shutdown + * @return True if redirectTo was set to a new address, false if redirectTo was not modified + */ + bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload); + + /** + * @param ip Address to check + * @return True if this is a cluster frontplane address (excluding our addresses) + */ + bool isClusterPeerFrontplane(const InetAddress &ip) const; + + /** + * Fill out ZT_ClusterStatus structure (from core API) + * + * @param status Reference to structure to hold result (anything there is replaced) + */ + void status(ZT_ClusterStatus &status) const; + +private: + void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len); + void _flush(uint16_t memberId); + + void _doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep); + void _doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep); + + // These are initialized in the constructor and remain immutable ------------ + uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; + unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; + const RuntimeEnvironment *RR; + _ClusterSendQueue *const _sendQueue; + void (*_sendFunction)(void *,unsigned int,const void *,unsigned int); + void *_sendFunctionArg; + int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *); + void *_addressToLocationFunctionArg; + const int32_t _x; + const int32_t _y; + const int32_t _z; + const uint16_t _id; + const std::vector _zeroTierPhysicalEndpoints; + // end immutable fields ----------------------------------------------------- + + struct _Member + { + unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; + + uint64_t lastReceivedAliveAnnouncement; + uint64_t lastAnnouncedAliveTo; + + uint64_t load; + uint64_t peers; + int32_t x,y,z; + + std::vector zeroTierPhysicalEndpoints; + + Buffer q; + + Mutex lock; + + inline void clear() + { + lastReceivedAliveAnnouncement = 0; + lastAnnouncedAliveTo = 0; + load = 0; + peers = 0; + x = 0; + y = 0; + z = 0; + zeroTierPhysicalEndpoints.clear(); + q.clear(); + } + + _Member() { this->clear(); } + ~_Member() { Utils::burn(key,sizeof(key)); } + }; + _Member *const _members; + + std::vector _memberIds; + Mutex _memberIds_m; + + struct _RemotePeer + { + _RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {} + ~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); } + uint64_t lastHavePeerReceived; + uint64_t lastSentWantPeer; + uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement + }; + std::map< std::pair,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here + Mutex _remotePeers_m; + + uint64_t _lastFlushed; + uint64_t _lastCleanedRemotePeers; + uint64_t _lastCleanedQueue; +}; + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +#endif diff --git a/node/Cluster.cpp b/node/Cluster.cpp deleted file mode 100644 index 119aec29..00000000 --- a/node/Cluster.cpp +++ /dev/null @@ -1,1042 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifdef ZT_ENABLE_CLUSTER - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "../version.h" - -#include "Cluster.hpp" -#include "RuntimeEnvironment.hpp" -#include "MulticastGroup.hpp" -#include "CertificateOfMembership.hpp" -#include "Salsa20.hpp" -#include "Poly1305.hpp" -#include "Identity.hpp" -#include "Topology.hpp" -#include "Packet.hpp" -#include "Switch.hpp" -#include "Node.hpp" -#include "Network.hpp" -#include "Array.hpp" - -namespace ZeroTier { - -static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2) - throw() -{ - double dx = ((double)x2 - (double)x1); - double dy = ((double)y2 - (double)y1); - double dz = ((double)z2 - (double)z1); - return sqrt((dx * dx) + (dy * dy) + (dz * dz)); -} - -// An entry in _ClusterSendQueue -struct _ClusterSendQueueEntry -{ - uint64_t timestamp; - Address fromPeerAddress; - Address toPeerAddress; - // if we ever support larger transport MTUs this must be increased - unsigned char data[ZT_CLUSTER_SEND_QUEUE_DATA_MAX]; - unsigned int len; - bool unite; -}; - -// A multi-index map with entry memory pooling -- this allows our queue to -// be O(log(N)) and is complex enough that it makes the code a lot cleaner -// to break it out from Cluster. -class _ClusterSendQueue -{ -public: - _ClusterSendQueue() : - _poolCount(0) {} - ~_ClusterSendQueue() {} // memory is automatically freed when _chunks is destroyed - - inline void enqueue(uint64_t now,const Address &from,const Address &to,const void *data,unsigned int len,bool unite) - { - if (len > ZT_CLUSTER_SEND_QUEUE_DATA_MAX) - return; - - Mutex::Lock _l(_lock); - - // Delete oldest queue entry for this sender if this enqueue() would take them over the per-sender limit - { - std::set< std::pair >::iterator qi(_bySrc.lower_bound(std::pair(from,(_ClusterSendQueueEntry *)0))); - std::set< std::pair >::iterator oldest(qi); - unsigned long countForSender = 0; - while ((qi != _bySrc.end())&&(qi->first == from)) { - if (qi->second->timestamp < oldest->second->timestamp) - oldest = qi; - ++countForSender; - ++qi; - } - if (countForSender >= ZT_CLUSTER_MAX_QUEUE_PER_SENDER) { - _byDest.erase(std::pair(oldest->second->toPeerAddress,oldest->second)); - _pool[_poolCount++] = oldest->second; - _bySrc.erase(oldest); - } - } - - _ClusterSendQueueEntry *e; - if (_poolCount > 0) { - e = _pool[--_poolCount]; - } else { - if (_chunks.size() >= ZT_CLUSTER_MAX_QUEUE_CHUNKS) - return; // queue is totally full! - _chunks.push_back(Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE>()); - e = &(_chunks.back().data[0]); - for(unsigned int i=1;itimestamp = now; - e->fromPeerAddress = from; - e->toPeerAddress = to; - memcpy(e->data,data,len); - e->len = len; - e->unite = unite; - - _bySrc.insert(std::pair(from,e)); - _byDest.insert(std::pair(to,e)); - } - - inline void expire(uint64_t now) - { - Mutex::Lock _l(_lock); - for(std::set< std::pair >::iterator qi(_bySrc.begin());qi!=_bySrc.end();) { - if ((now - qi->second->timestamp) > ZT_CLUSTER_QUEUE_EXPIRATION) { - _byDest.erase(std::pair(qi->second->toPeerAddress,qi->second)); - _pool[_poolCount++] = qi->second; - _bySrc.erase(qi++); - } else ++qi; - } - } - - /** - * Get and dequeue entries for a given destination address - * - * After use these entries must be returned with returnToPool()! - * - * @param dest Destination address - * @param results Array to fill with results - * @param maxResults Size of results[] in pointers - * @return Number of actual results returned - */ - inline unsigned int getByDest(const Address &dest,_ClusterSendQueueEntry **results,unsigned int maxResults) - { - unsigned int count = 0; - Mutex::Lock _l(_lock); - std::set< std::pair >::iterator qi(_byDest.lower_bound(std::pair(dest,(_ClusterSendQueueEntry *)0))); - while ((qi != _byDest.end())&&(qi->first == dest)) { - _bySrc.erase(std::pair(qi->second->fromPeerAddress,qi->second)); - results[count++] = qi->second; - if (count == maxResults) - break; - _byDest.erase(qi++); - } - return count; - } - - /** - * Return entries to pool after use - * - * @param entries Array of entries - * @param count Number of entries - */ - inline void returnToPool(_ClusterSendQueueEntry **entries,unsigned int count) - { - Mutex::Lock _l(_lock); - for(unsigned int i=0;i > _chunks; - _ClusterSendQueueEntry *_pool[ZT_CLUSTER_QUEUE_CHUNK_SIZE * ZT_CLUSTER_MAX_QUEUE_CHUNKS]; - unsigned long _poolCount; - std::set< std::pair > _bySrc; - std::set< std::pair > _byDest; - Mutex _lock; -}; - -Cluster::Cluster( - const RuntimeEnvironment *renv, - uint16_t id, - const std::vector &zeroTierPhysicalEndpoints, - int32_t x, - int32_t y, - int32_t z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg) : - RR(renv), - _sendQueue(new _ClusterSendQueue()), - _sendFunction(sendFunction), - _sendFunctionArg(sendFunctionArg), - _addressToLocationFunction(addressToLocationFunction), - _addressToLocationFunctionArg(addressToLocationFunctionArg), - _x(x), - _y(y), - _z(z), - _id(id), - _zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints), - _members(new _Member[ZT_CLUSTER_MAX_MEMBERS]), - _lastFlushed(0), - _lastCleanedRemotePeers(0), - _lastCleanedQueue(0) -{ - uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; - - // Generate master secret by hashing the secret from our Identity key pair - RR->identity.sha512PrivateKey(_masterSecret); - - // Generate our inbound message key, which is the master secret XORed with our ID and hashed twice - memcpy(stmp,_masterSecret,sizeof(stmp)); - stmp[0] ^= Utils::hton(id); - SHA512::hash(stmp,stmp,sizeof(stmp)); - SHA512::hash(stmp,stmp,sizeof(stmp)); - memcpy(_key,stmp,sizeof(_key)); - Utils::burn(stmp,sizeof(stmp)); -} - -Cluster::~Cluster() -{ - Utils::burn(_masterSecret,sizeof(_masterSecret)); - Utils::burn(_key,sizeof(_key)); - delete [] _members; - delete _sendQueue; -} - -void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len) -{ - Buffer dmsg; - { - // FORMAT: <[16] iv><[8] MAC><... data> - if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH)) - return; - - // 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV - char keytmp[32]; - memcpy(keytmp,_key,32); - for(int i=0;i<8;++i) - keytmp[i] ^= reinterpret_cast(msg)[i]; - Salsa20 s20(keytmp,reinterpret_cast(msg) + 8); - Utils::burn(keytmp,sizeof(keytmp)); - - // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") - char polykey[ZT_POLY1305_KEY_LEN]; - memset(polykey,0,sizeof(polykey)); - s20.crypt12(polykey,polykey,sizeof(polykey)); - - // Compute 16-byte MAC - char mac[ZT_POLY1305_MAC_LEN]; - Poly1305::compute(mac,reinterpret_cast(msg) + 24,len - 24,polykey); - - // Check first 8 bytes of MAC against 64-bit MAC in stream - if (!Utils::secureEq(mac,reinterpret_cast(msg) + 16,8)) - return; - - // Decrypt! - dmsg.setSize(len - 24); - s20.crypt12(reinterpret_cast(msg) + 24,const_cast(dmsg.data()),dmsg.size()); - } - - if (dmsg.size() < 4) - return; - const uint16_t fromMemberId = dmsg.at(0); - unsigned int ptr = 2; - if (fromMemberId == _id) // sanity check: we don't talk to ourselves - return; - const uint16_t toMemberId = dmsg.at(ptr); - ptr += 2; - if (toMemberId != _id) // sanity check: message not for us? - return; - - { // make sure sender is actually considered a member - Mutex::Lock _l3(_memberIds_m); - if (std::find(_memberIds.begin(),_memberIds.end(),fromMemberId) == _memberIds.end()) - return; - } - - try { - while (ptr < dmsg.size()) { - const unsigned int mlen = dmsg.at(ptr); ptr += 2; - const unsigned int nextPtr = ptr + mlen; - if (nextPtr > dmsg.size()) - break; - - int mtype = -1; - try { - switch((StateMessageType)(mtype = (int)dmsg[ptr++])) { - default: - break; - - case CLUSTER_MESSAGE_ALIVE: { - _Member &m = _members[fromMemberId]; - Mutex::Lock mlck(m.lock); - ptr += 7; // skip version stuff, not used yet - m.x = dmsg.at(ptr); ptr += 4; - m.y = dmsg.at(ptr); ptr += 4; - m.z = dmsg.at(ptr); ptr += 4; - ptr += 8; // skip local clock, not used - m.load = dmsg.at(ptr); ptr += 8; - m.peers = dmsg.at(ptr); ptr += 8; - ptr += 8; // skip flags, unused -#ifdef ZT_TRACE - std::string addrs; -#endif - unsigned int physicalAddressCount = dmsg[ptr++]; - m.zeroTierPhysicalEndpoints.clear(); - for(unsigned int i=0;i 0) - addrs.push_back(','); - addrs.append(m.zeroTierPhysicalEndpoints.back().toString()); - } -#endif - } -#ifdef ZT_TRACE - if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) { - TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str()); - } -#endif - m.lastReceivedAliveAnnouncement = RR->node->now(); - } break; - - case CLUSTER_MESSAGE_HAVE_PEER: { - Identity id; - ptr += id.deserialize(dmsg,ptr); - if (id) { - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair(id.address(),(unsigned int)fromMemberId)]; - if (!rp.lastHavePeerReceived) { - RR->topology->saveIdentity((void *)0,id); - RR->identity.agree(id,rp.key,ZT_PEER_SECRET_KEY_LENGTH); - } - rp.lastHavePeerReceived = RR->node->now(); - } - - _ClusterSendQueueEntry *q[16384]; // 16384 is "tons" - unsigned int qc = _sendQueue->getByDest(id.address(),q,16384); - for(unsigned int i=0;irelayViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite); - _sendQueue->returnToPool(q,qc); - - TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc); - } - } break; - - case CLUSTER_MESSAGE_WANT_PEER: { - const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - SharedPtr peer(RR->topology->getPeerNoCache(zeroTierAddress)); - if ( (peer) && (peer->hasLocalClusterOptimalPath(RR->node->now())) ) { - Buffer<1024> buf; - peer->identity().serialize(buf); - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); - } - } break; - - case CLUSTER_MESSAGE_REMOTE_PACKET: { - const unsigned int plen = dmsg.at(ptr); ptr += 2; - if (plen) { - Packet remotep(dmsg.field(ptr,plen),plen); ptr += plen; - //TRACE("remote %s from %s via %u (%u bytes)",Packet::verbString(remotep.verb()),remotep.source().toString().c_str(),fromMemberId,plen); - switch(remotep.verb()) { - case Packet::VERB_WHOIS: _doREMOTE_WHOIS(fromMemberId,remotep); break; - case Packet::VERB_MULTICAST_GATHER: _doREMOTE_MULTICAST_GATHER(fromMemberId,remotep); break; - default: break; // ignore things we don't care about across cluster - } - } - } break; - - case CLUSTER_MESSAGE_PROXY_UNITE: { - const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const unsigned int numRemotePeerPaths = dmsg[ptr++]; - InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max - for(unsigned int i=0;inode->now(); - SharedPtr localPeer(RR->topology->getPeerNoCache(localPeerAddress)); - if ((localPeer)&&(numRemotePeerPaths > 0)) { - InetAddress bestLocalV4,bestLocalV6; - localPeer->getRendezvousAddresses(now,bestLocalV4,bestLocalV6); - - InetAddress bestRemoteV4,bestRemoteV6; - for(unsigned int i=0;iidentity.address(),Packet::VERB_RENDEZVOUS); - rendezvousForLocal.append((uint8_t)0); - remotePeerAddress.appendTo(rendezvousForLocal); - - Buffer<2048> rendezvousForRemote; - remotePeerAddress.appendTo(rendezvousForRemote); - rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS); - rendezvousForRemote.addSize(2); // space for actual packet payload length - rendezvousForRemote.append((uint8_t)0); // flags == 0 - localPeerAddress.appendTo(rendezvousForRemote); - - bool haveMatch = false; - if ((bestLocalV6)&&(bestRemoteV6)) { - haveMatch = true; - - rendezvousForLocal.append((uint16_t)bestRemoteV6.port()); - rendezvousForLocal.append((uint8_t)16); - rendezvousForLocal.append(bestRemoteV6.rawIpData(),16); - - rendezvousForRemote.append((uint16_t)bestLocalV6.port()); - rendezvousForRemote.append((uint8_t)16); - rendezvousForRemote.append(bestLocalV6.rawIpData(),16); - rendezvousForRemote.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 16)); - } else if ((bestLocalV4)&&(bestRemoteV4)) { - haveMatch = true; - - rendezvousForLocal.append((uint16_t)bestRemoteV4.port()); - rendezvousForLocal.append((uint8_t)4); - rendezvousForLocal.append(bestRemoteV4.rawIpData(),4); - - rendezvousForRemote.append((uint16_t)bestLocalV4.port()); - rendezvousForRemote.append((uint8_t)4); - rendezvousForRemote.append(bestLocalV4.rawIpData(),4); - rendezvousForRemote.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 4)); - } - - if (haveMatch) { - { - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size()); - } - RR->sw->send((void *)0,rendezvousForLocal,true); - } - } - } break; - - case CLUSTER_MESSAGE_PROXY_SEND: { - const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH; - const Packet::Verb verb = (Packet::Verb)dmsg[ptr++]; - const unsigned int len = dmsg.at(ptr); ptr += 2; - Packet outp(rcpt,RR->identity.address(),verb); - outp.append(dmsg.field(ptr,len),len); ptr += len; - RR->sw->send((void *)0,outp,true); - //TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len); - } break; - - case CLUSTER_MESSAGE_NETWORK_CONFIG: { - const SharedPtr network(RR->node->network(dmsg.at(ptr))); - if (network) { - // Copy into a Packet just to conform to Network API. Eventually - // will want to refactor. - network->handleConfigChunk((void *)0,0,Address(),Buffer(dmsg),ptr); - } - } break; - } - } catch ( ... ) { - TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype); - // drop invalids - } - - ptr = nextPtr; - } - } catch ( ... ) { - TRACE("invalid message (outer loop), discarding"); - // drop invalids - } -} - -void Cluster::broadcastHavePeer(const Identity &id) -{ - Buffer<1024> buf; - id.serialize(buf); - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size()); - } -} - -void Cluster::broadcastNetworkConfigChunk(const void *chunk,unsigned int len) -{ - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_NETWORK_CONFIG,chunk,len); - } -} - -int Cluster::checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret) -{ - const uint64_t now = RR->node->now(); - mostRecentTs = 0; - int mostRecentMemberId = -1; - { - Mutex::Lock _l2(_remotePeers_m); - std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); - for(;;) { - if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) - break; - else if (rpe->second.lastHavePeerReceived > mostRecentTs) { - mostRecentTs = rpe->second.lastHavePeerReceived; - memcpy(peerSecret,rpe->second.key,ZT_PEER_SECRET_KEY_LENGTH); - mostRecentMemberId = (int)rpe->first.second; - } - ++rpe; - } - } - - const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; - if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { - if (ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT) - mostRecentMemberId = -1; - - bool sendWantPeer = true; - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; - if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { - rp.lastSentWantPeer = now; - } else { - sendWantPeer = false; // don't flood WANT_PEER - } - } - if (sendWantPeer) { - char tmp[ZT_ADDRESS_LENGTH]; - toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); - { - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); - } - } - } - } - - return mostRecentMemberId; -} - -bool Cluster::sendViaCluster(int mostRecentMemberId,const Address &toPeerAddress,const void *data,unsigned int len) -{ - if ((mostRecentMemberId < 0)||(mostRecentMemberId >= ZT_CLUSTER_MAX_MEMBERS)) // sanity check - return false; - Mutex::Lock _l2(_members[mostRecentMemberId].lock); - for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { - for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { - if (i1->ss_family == i2->ss_family) { - TRACE("sendViaCluster sending %u bytes to %s by way of %u (%s->%s)",len,toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket((void *)0,*i1,*i2,data,len); - return true; - } - } - } - return false; -} - -void Cluster::relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite) -{ - if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check - return; - - const uint64_t now = RR->node->now(); - - uint64_t mostRecentTs = 0; - int mostRecentMemberId = -1; - { - Mutex::Lock _l2(_remotePeers_m); - std::map< std::pair,_RemotePeer >::const_iterator rpe(_remotePeers.lower_bound(std::pair(toPeerAddress,0))); - for(;;) { - if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress)) - break; - else if (rpe->second.lastHavePeerReceived > mostRecentTs) { - mostRecentTs = rpe->second.lastHavePeerReceived; - mostRecentMemberId = (int)rpe->first.second; - } - ++rpe; - } - } - - const uint64_t ageOfMostRecentHavePeerAnnouncement = now - mostRecentTs; - if (ageOfMostRecentHavePeerAnnouncement >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) { - // Enqueue and wait if peer seems alive, but do WANT_PEER to refresh homing - const bool enqueueAndWait = ((ageOfMostRecentHavePeerAnnouncement >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId < 0)); - - // Poll everyone with WANT_PEER if the age of our most recent entry is - // approaching expiration (or has expired, or does not exist). - bool sendWantPeer = true; - { - Mutex::Lock _l(_remotePeers_m); - _RemotePeer &rp = _remotePeers[std::pair(toPeerAddress,(unsigned int)_id)]; - if ((now - rp.lastSentWantPeer) >= ZT_CLUSTER_WANT_PEER_EVERY) { - rp.lastSentWantPeer = now; - } else { - sendWantPeer = false; // don't flood WANT_PEER - } - } - if (sendWantPeer) { - char tmp[ZT_ADDRESS_LENGTH]; - toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH); - { - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH); - } - } - } - - // If there isn't a good place to send via, then enqueue this for retrying - // later and return after having broadcasted a WANT_PEER. - if (enqueueAndWait) { - TRACE("relayViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str()); - _sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite); - return; - } - } - - if (mostRecentMemberId >= 0) { - Buffer<1024> buf; - if (unite) { - InetAddress v4,v6; - if (fromPeerAddress) { - SharedPtr fromPeer(RR->topology->getPeerNoCache(fromPeerAddress)); - if (fromPeer) - fromPeer->getRendezvousAddresses(now,v4,v6); - } - uint8_t addrCount = 0; - if (v4) - ++addrCount; - if (v6) - ++addrCount; - if (addrCount) { - toPeerAddress.appendTo(buf); - fromPeerAddress.appendTo(buf); - buf.append(addrCount); - if (v4) - v4.serialize(buf); - if (v6) - v6.serialize(buf); - } - } - - { - Mutex::Lock _l2(_members[mostRecentMemberId].lock); - if (buf.size() > 0) - _send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size()); - - for(std::vector::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) { - for(std::vector::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) { - if (i1->ss_family == i2->ss_family) { - TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str()); - RR->node->putPacket((void *)0,*i1,*i2,data,len); - return; - } - } - } - - TRACE("relayViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId); - } - } -} - -void Cluster::sendDistributedQuery(const Packet &pkt) -{ - Buffer<4096> buf; - buf.append((uint16_t)pkt.size()); - buf.append(pkt.data(),pkt.size()); - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - _send(*mid,CLUSTER_MESSAGE_REMOTE_PACKET,buf.data(),buf.size()); - } -} - -void Cluster::doPeriodicTasks() -{ - const uint64_t now = RR->node->now(); - - if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) { - _lastFlushed = now; - - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - - if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) { - _members[*mid].lastAnnouncedAliveTo = now; - - Buffer<2048> alive; - alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR); - alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR); - alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); - alive.append((uint8_t)ZT_PROTO_VERSION); - if (_addressToLocationFunction) { - alive.append((int32_t)_x); - alive.append((int32_t)_y); - alive.append((int32_t)_z); - } else { - alive.append((int32_t)0); - alive.append((int32_t)0); - alive.append((int32_t)0); - } - alive.append((uint64_t)now); - alive.append((uint64_t)0); // TODO: compute and send load average - alive.append((uint64_t)RR->topology->countActive(now)); - alive.append((uint64_t)0); // unused/reserved flags - alive.append((uint8_t)_zeroTierPhysicalEndpoints.size()); - for(std::vector::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe) - pe->serialize(alive); - _send(*mid,CLUSTER_MESSAGE_ALIVE,alive.data(),alive.size()); - } - - _flush(*mid); - } - } - - if ((now - _lastCleanedRemotePeers) >= (ZT_PEER_ACTIVITY_TIMEOUT * 2)) { - _lastCleanedRemotePeers = now; - - Mutex::Lock _l(_remotePeers_m); - for(std::map< std::pair,_RemotePeer >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) { - if ((now - rp->second.lastHavePeerReceived) >= ZT_PEER_ACTIVITY_TIMEOUT) - _remotePeers.erase(rp++); - else ++rp; - } - } - - if ((now - _lastCleanedQueue) >= ZT_CLUSTER_QUEUE_EXPIRATION) { - _lastCleanedQueue = now; - _sendQueue->expire(now); - } -} - -void Cluster::addMember(uint16_t memberId) -{ - if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id)) - return; - - Mutex::Lock _l2(_members[memberId].lock); - - { - Mutex::Lock _l(_memberIds_m); - if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end()) - return; - _memberIds.push_back(memberId); - std::sort(_memberIds.begin(),_memberIds.end()); - } - - _members[memberId].clear(); - - // Generate this member's message key from the master and its ID - uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; - memcpy(stmp,_masterSecret,sizeof(stmp)); - stmp[0] ^= Utils::hton(memberId); - SHA512::hash(stmp,stmp,sizeof(stmp)); - SHA512::hash(stmp,stmp,sizeof(stmp)); - memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key)); - Utils::burn(stmp,sizeof(stmp)); - - // Prepare q - _members[memberId].q.clear(); - char iv[16]; - Utils::getSecureRandom(iv,16); - _members[memberId].q.append(iv,16); - _members[memberId].q.addSize(8); // room for MAC - _members[memberId].q.append((uint16_t)_id); - _members[memberId].q.append((uint16_t)memberId); -} - -void Cluster::removeMember(uint16_t memberId) -{ - Mutex::Lock _l(_memberIds_m); - std::vector newMemberIds; - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - if (*mid != memberId) - newMemberIds.push_back(*mid); - } - _memberIds = newMemberIds; -} - -bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload) -{ - if (_addressToLocationFunction) { - // Pick based on location if it can be determined - int px = 0,py = 0,pz = 0; - if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast(&peerPhysicalAddress),&px,&py,&pz) == 0) { - TRACE("no geolocation data for %s",peerPhysicalAddress.toIpString().c_str()); - return false; - } - - // Find member closest to this peer - const uint64_t now = RR->node->now(); - std::vector best; - const double currentDistance = _dist3d(_x,_y,_z,px,py,pz); - double bestDistance = (offload ? 2147483648.0 : currentDistance); -#ifdef ZT_TRACE - unsigned int bestMember = _id; -#endif - { - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - _Member &m = _members[*mid]; - Mutex::Lock _ml(m.lock); - - // Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to - if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) { - const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz); - if (mdist < bestDistance) { - bestDistance = mdist; -#ifdef ZT_TRACE - bestMember = *mid; -#endif - best = m.zeroTierPhysicalEndpoints; - } - } - } - } - - // Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family - for(std::vector::const_iterator a(best.begin());a!=best.end();++a) { - if (a->ss_family == peerPhysicalAddress.ss_family) { - TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str()); - redirectTo = *a; - return true; - } - } - TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance); - return false; - } else { - // TODO: pick based on load if no location info? - return false; - } -} - -bool Cluster::isClusterPeerFrontplane(const InetAddress &ip) const -{ - Mutex::Lock _l(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - Mutex::Lock _l2(_members[*mid].lock); - for(std::vector::const_iterator i2(_members[*mid].zeroTierPhysicalEndpoints.begin());i2!=_members[*mid].zeroTierPhysicalEndpoints.end();++i2) { - if (ip == *i2) - return true; - } - } - return false; -} - -void Cluster::status(ZT_ClusterStatus &status) const -{ - const uint64_t now = RR->node->now(); - memset(&status,0,sizeof(ZT_ClusterStatus)); - - status.myId = _id; - - { - ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); - s->id = _id; - s->alive = 1; - s->x = _x; - s->y = _y; - s->z = _z; - s->load = 0; // TODO - s->peers = RR->topology->countActive(now); - for(std::vector::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) { - if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check - break; - memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); - } - } - - { - Mutex::Lock _l1(_memberIds_m); - for(std::vector::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) { - if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check - break; - - _Member &m = _members[*mid]; - Mutex::Lock ml(m.lock); - - ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]); - s->id = *mid; - s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement)); - s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0; - s->x = m.x; - s->y = m.y; - s->z = m.z; - s->load = m.load; - s->peers = m.peers; - for(std::vector::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) { - if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check - break; - memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage)); - } - } - } -} - -void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len) -{ - if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check - return; - _Member &m = _members[memberId]; - // assumes m.lock is locked! - if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH) - _flush(memberId); - m.q.append((uint16_t)(len + 1)); - m.q.append((uint8_t)type); - m.q.append(msg,len); -} - -void Cluster::_flush(uint16_t memberId) -{ - _Member &m = _members[memberId]; - // assumes m.lock is locked! - if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID - // Create key from member's key and IV - char keytmp[32]; - memcpy(keytmp,m.key,32); - for(int i=0;i<8;++i) - keytmp[i] ^= m.q[i]; - Salsa20 s20(keytmp,m.q.field(8,8)); - Utils::burn(keytmp,sizeof(keytmp)); - - // One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard") - char polykey[ZT_POLY1305_KEY_LEN]; - memset(polykey,0,sizeof(polykey)); - s20.crypt12(polykey,polykey,sizeof(polykey)); - - // Encrypt m.q in place - s20.crypt12(reinterpret_cast(m.q.data()) + 24,const_cast(reinterpret_cast(m.q.data())) + 24,m.q.size() - 24); - - // Add MAC for authentication (encrypt-then-MAC) - char mac[ZT_POLY1305_MAC_LEN]; - Poly1305::compute(mac,reinterpret_cast(m.q.data()) + 24,m.q.size() - 24,polykey); - memcpy(m.q.field(16,8),mac,8); - - // Send! - _sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size()); - - // Prepare for more - m.q.clear(); - char iv[16]; - Utils::getSecureRandom(iv,16); - m.q.append(iv,16); - m.q.addSize(8); // room for MAC - m.q.append((uint16_t)_id); // from member ID - m.q.append((uint16_t)memberId); // to member ID - } -} - -void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep) -{ - if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) { - Identity queried(RR->topology->getIdentity((void *)0,Address(remotep.payload(),ZT_ADDRESS_LENGTH))); - if (queried) { - Buffer<1024> routp; - remotep.source().appendTo(routp); - routp.append((uint8_t)Packet::VERB_OK); - routp.addSize(2); // space for length - routp.append((uint8_t)Packet::VERB_WHOIS); - routp.append(remotep.packetId()); - queried.serialize(routp); - routp.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); - - TRACE("responding to remote WHOIS from %s @ %u with identity of %s",remotep.source().toString().c_str(),(unsigned int)fromMemberId,queried.address().toString().c_str()); - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); - } - } -} - -void Cluster::_doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep) -{ - const uint64_t nwid = remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); - const MulticastGroup mg(MAC(remotep.field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); - unsigned int gatherLimit = remotep.at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); - const Address remotePeerAddress(remotep.source()); - - if (gatherLimit) { - Buffer routp; - remotePeerAddress.appendTo(routp); - routp.append((uint8_t)Packet::VERB_OK); - routp.addSize(2); // space for length - routp.append((uint8_t)Packet::VERB_MULTICAST_GATHER); - routp.append(remotep.packetId()); - routp.append(nwid); - mg.mac().appendTo(routp); - routp.append((uint32_t)mg.adi()); - - if (gatherLimit > ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5)) - gatherLimit = ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5); - if (RR->mc->gather(remotePeerAddress,nwid,mg,routp,gatherLimit)) { - routp.setAt(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3)); - - TRACE("responding to remote MULTICAST_GATHER from %s @ %u with %u bytes",remotePeerAddress.toString().c_str(),(unsigned int)fromMemberId,routp.size()); - Mutex::Lock _l2(_members[fromMemberId].lock); - _send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size()); - } - } -} - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER diff --git a/node/Cluster.hpp b/node/Cluster.hpp deleted file mode 100644 index 74b091f5..00000000 --- a/node/Cluster.hpp +++ /dev/null @@ -1,463 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTER_HPP -#define ZT_CLUSTER_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include - -#include "Constants.hpp" -#include "../include/ZeroTierOne.h" -#include "Address.hpp" -#include "InetAddress.hpp" -#include "SHA512.hpp" -#include "Utils.hpp" -#include "Buffer.hpp" -#include "Mutex.hpp" -#include "SharedPtr.hpp" -#include "Hashtable.hpp" -#include "Packet.hpp" -#include "SharedPtr.hpp" - -/** - * Timeout for cluster members being considered "alive" - * - * A cluster member is considered dead and will no longer have peers - * redirected to it if we have not heard a heartbeat in this long. - */ -#define ZT_CLUSTER_TIMEOUT 5000 - -/** - * Desired period between doPeriodicTasks() in milliseconds - */ -#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 20 - -/** - * How often to flush outgoing message queues (maximum interval) - */ -#define ZT_CLUSTER_FLUSH_PERIOD ZT_CLUSTER_PERIODIC_TASK_PERIOD - -/** - * Maximum number of queued outgoing packets per sender address - */ -#define ZT_CLUSTER_MAX_QUEUE_PER_SENDER 16 - -/** - * Expiration time for send queue entries - */ -#define ZT_CLUSTER_QUEUE_EXPIRATION 3000 - -/** - * Chunk size for allocating queue entries - * - * Queue entries are allocated in chunks of this many and are added to a pool. - * ZT_CLUSTER_MAX_QUEUE_GLOBAL must be evenly divisible by this. - */ -#define ZT_CLUSTER_QUEUE_CHUNK_SIZE 32 - -/** - * Maximum number of chunks to ever allocate - * - * This is a global sanity limit to prevent resource exhaustion attacks. It - * works out to about 600mb of RAM. You'll never see this on a normal edge - * node. We're unlikely to see this on a root server unless someone is DOSing - * us. In that case cluster relaying will be affected but other functions - * should continue to operate normally. - */ -#define ZT_CLUSTER_MAX_QUEUE_CHUNKS 8194 - -/** - * Max data per queue entry - */ -#define ZT_CLUSTER_SEND_QUEUE_DATA_MAX 1500 - -/** - * We won't send WANT_PEER to other members more than every (ms) per recipient - */ -#define ZT_CLUSTER_WANT_PEER_EVERY 1000 - -namespace ZeroTier { - -class RuntimeEnvironment; -class MulticastGroup; -class Peer; -class Identity; - -// Internal class implemented inside Cluster.cpp -class _ClusterSendQueue; - -/** - * Multi-homing cluster state replication and packet relaying - * - * Multi-homing means more than one node sharing the same ZeroTier identity. - * There is nothing in the protocol to prevent this, but to make it work well - * requires the devices sharing an identity to cooperate and share some - * information. - * - * There are three use cases we want to fulfill: - * - * (1) Multi-homing of root servers with handoff for efficient routing, - * HA, and load balancing across many commodity nodes. - * (2) Multi-homing of network controllers for the same reason. - * (3) Multi-homing of nodes on virtual networks, such as domain servers - * and other important endpoints. - * - * These use cases are in order of escalating difficulty. The initial - * version of Cluster is aimed at satisfying the first, though you are - * free to try #2 and #3. - */ -class Cluster -{ -public: - /** - * State message types - */ - enum StateMessageType - { - CLUSTER_MESSAGE_NOP = 0, - - /** - * This cluster member is alive: - * <[2] version minor> - * <[2] version major> - * <[2] version revision> - * <[1] protocol version> - * <[4] X location (signed 32-bit)> - * <[4] Y location (signed 32-bit)> - * <[4] Z location (signed 32-bit)> - * <[8] local clock at this member> - * <[8] load average> - * <[8] number of peers> - * <[8] flags (currently unused, must be zero)> - * <[1] number of preferred ZeroTier endpoints> - * <[...] InetAddress(es) of preferred ZeroTier endpoint(s)> - * - * Cluster members constantly broadcast an alive heartbeat and will only - * receive peer redirects if they've done so within the timeout. - */ - CLUSTER_MESSAGE_ALIVE = 1, - - /** - * Cluster member has this peer: - * <[...] serialized identity of peer> - * - * This is typically sent in response to WANT_PEER but can also be pushed - * to prepopulate if this makes sense. - */ - CLUSTER_MESSAGE_HAVE_PEER = 2, - - /** - * Cluster member wants this peer: - * <[5] ZeroTier address of peer> - * - * Members that have a direct link to this peer will respond with - * HAVE_PEER. - */ - CLUSTER_MESSAGE_WANT_PEER = 3, - - /** - * A remote packet that we should also possibly respond to: - * <[2] 16-bit length of remote packet> - * <[...] remote packet payload> - * - * Cluster members may relay requests by relaying the request packet. - * These may include requests such as WHOIS and MULTICAST_GATHER. The - * packet must be already decrypted, decompressed, and authenticated. - * - * This can only be used for small request packets as per the cluster - * message size limit, but since these are the only ones in question - * this is fine. - * - * If a response is generated it is sent via PROXY_SEND. - */ - CLUSTER_MESSAGE_REMOTE_PACKET = 4, - - /** - * Request that VERB_RENDEZVOUS be sent to a peer that we have: - * <[5] ZeroTier address of peer on recipient's side> - * <[5] ZeroTier address of peer on sender's side> - * <[1] 8-bit number of sender's peer's active path addresses> - * <[...] series of serialized InetAddresses of sender's peer's paths> - * - * This requests that we perform NAT-t introduction between a peer that - * we have and one on the sender's side. The sender furnishes contact - * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours - * directly and with PROXY_SEND to theirs. - */ - CLUSTER_MESSAGE_PROXY_UNITE = 5, - - /** - * Request that a cluster member send a packet to a locally-known peer: - * <[5] ZeroTier address of recipient> - * <[1] packet verb> - * <[2] length of packet payload> - * <[...] packet payload> - * - * This differs from RELAY in that it requests the receiving cluster - * member to actually compose a ZeroTier Packet from itself to the - * provided recipient. RELAY simply says "please forward this blob." - * RELAY is used to implement peer-to-peer relaying with RENDEZVOUS, - * while PROXY_SEND is used to implement proxy sending (which right - * now is only used to send RENDEZVOUS). - */ - CLUSTER_MESSAGE_PROXY_SEND = 6, - - /** - * Replicate a network config for a network we belong to: - * <[...] network config chunk> - * - * This is used by clusters to avoid every member having to query - * for the same netconf for networks all members belong to. - * - * The first field of a network config chunk is the network ID, - * so this can be checked to look up the network on receipt. - */ - CLUSTER_MESSAGE_NETWORK_CONFIG = 7 - }; - - /** - * Construct a new cluster - */ - Cluster( - const RuntimeEnvironment *renv, - uint16_t id, - const std::vector &zeroTierPhysicalEndpoints, - int32_t x, - int32_t y, - int32_t z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg); - - ~Cluster(); - - /** - * @return This cluster member's ID - */ - inline uint16_t id() const throw() { return _id; } - - /** - * Handle an incoming intra-cluster message - * - * @param data Message data - * @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH) - */ - void handleIncomingStateMessage(const void *msg,unsigned int len); - - /** - * Broadcast that we have a given peer - * - * This should be done when new peers are first contacted. - * - * @param id Identity of peer - */ - void broadcastHavePeer(const Identity &id); - - /** - * Broadcast a network config chunk to other members of cluster - * - * @param chunk Chunk data - * @param len Length of chunk - */ - void broadcastNetworkConfigChunk(const void *chunk,unsigned int len); - - /** - * If the cluster has this peer, prepare the packet to send via cluster - * - * Note that outp is only armored (or modified at all) if the return value is a member ID. - * - * @param toPeerAddress Value of outp.destination(), simply to save additional lookup - * @param ts Result: set to time of last HAVE_PEER from the cluster - * @param peerSecret Result: Buffer to fill with peer secret on valid return value, must be at least ZT_PEER_SECRET_KEY_LENGTH bytes - * @return -1 if cluster does not know this peer, or a member ID to pass to sendViaCluster() - */ - int checkSendViaCluster(const Address &toPeerAddress,uint64_t &mostRecentTs,void *peerSecret); - - /** - * Send data via cluster front plane (packet head or fragment) - * - * @param haveMemberId Member ID that has this peer as returned by prepSendviaCluster() - * @param toPeerAddress Destination peer address - * @param data Packet or packet fragment data - * @param len Length of packet or fragment - * @return True if packet was sent (and outp was modified via armoring) - */ - bool sendViaCluster(int haveMemberId,const Address &toPeerAddress,const void *data,unsigned int len); - - /** - * Relay a packet via the cluster - * - * This is used in the outgoing packet and relaying logic in Switch to - * relay packets to other cluster members. It isn't PROXY_SEND-- that is - * used internally in Cluster to send responses to peer queries. - * - * @param fromPeerAddress Source peer address (if known, should be NULL for fragments) - * @param toPeerAddress Destination peer address - * @param data Packet or packet fragment data - * @param len Length of packet or fragment - * @param unite If true, also request proxy unite across cluster - */ - void relayViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite); - - /** - * Send a distributed query to other cluster members - * - * Some queries such as WHOIS or MULTICAST_GATHER need a response from other - * cluster members. Replies (if any) will be sent back to the peer via - * PROXY_SEND across the cluster. - * - * @param pkt Packet to distribute - */ - void sendDistributedQuery(const Packet &pkt); - - /** - * Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds. - */ - void doPeriodicTasks(); - - /** - * Add a member ID to this cluster - * - * @param memberId Member ID - */ - void addMember(uint16_t memberId); - - /** - * Remove a member ID from this cluster - * - * @param memberId Member ID to remove - */ - void removeMember(uint16_t memberId); - - /** - * Find a better cluster endpoint for this peer (if any) - * - * @param redirectTo InetAddress to be set to a better endpoint (if there is one) - * @param peerAddress Address of peer to (possibly) redirect - * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address()) - * @param offload Always redirect if possible -- can be used to offload peers during shutdown - * @return True if redirectTo was set to a new address, false if redirectTo was not modified - */ - bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload); - - /** - * @param ip Address to check - * @return True if this is a cluster frontplane address (excluding our addresses) - */ - bool isClusterPeerFrontplane(const InetAddress &ip) const; - - /** - * Fill out ZT_ClusterStatus structure (from core API) - * - * @param status Reference to structure to hold result (anything there is replaced) - */ - void status(ZT_ClusterStatus &status) const; - -private: - void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len); - void _flush(uint16_t memberId); - - void _doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep); - void _doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep); - - // These are initialized in the constructor and remain immutable ------------ - uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)]; - unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; - const RuntimeEnvironment *RR; - _ClusterSendQueue *const _sendQueue; - void (*_sendFunction)(void *,unsigned int,const void *,unsigned int); - void *_sendFunctionArg; - int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *); - void *_addressToLocationFunctionArg; - const int32_t _x; - const int32_t _y; - const int32_t _z; - const uint16_t _id; - const std::vector _zeroTierPhysicalEndpoints; - // end immutable fields ----------------------------------------------------- - - struct _Member - { - unsigned char key[ZT_PEER_SECRET_KEY_LENGTH]; - - uint64_t lastReceivedAliveAnnouncement; - uint64_t lastAnnouncedAliveTo; - - uint64_t load; - uint64_t peers; - int32_t x,y,z; - - std::vector zeroTierPhysicalEndpoints; - - Buffer q; - - Mutex lock; - - inline void clear() - { - lastReceivedAliveAnnouncement = 0; - lastAnnouncedAliveTo = 0; - load = 0; - peers = 0; - x = 0; - y = 0; - z = 0; - zeroTierPhysicalEndpoints.clear(); - q.clear(); - } - - _Member() { this->clear(); } - ~_Member() { Utils::burn(key,sizeof(key)); } - }; - _Member *const _members; - - std::vector _memberIds; - Mutex _memberIds_m; - - struct _RemotePeer - { - _RemotePeer() : lastHavePeerReceived(0),lastSentWantPeer(0) {} - ~_RemotePeer() { Utils::burn(key,ZT_PEER_SECRET_KEY_LENGTH); } - uint64_t lastHavePeerReceived; - uint64_t lastSentWantPeer; - uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; // secret key from identity agreement - }; - std::map< std::pair,_RemotePeer > _remotePeers; // we need ordered behavior and lower_bound here - Mutex _remotePeers_m; - - uint64_t _lastFlushed; - uint64_t _lastCleanedRemotePeers; - uint64_t _lastCleanedQueue; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif diff --git a/objects.mk b/objects.mk index fd25a3d3..c8231f08 100644 --- a/objects.mk +++ b/objects.mk @@ -3,7 +3,6 @@ CORE_OBJS=\ node/Capability.o \ node/CertificateOfMembership.o \ node/CertificateOfOwnership.o \ - node/Cluster.o \ node/Identity.o \ node/IncomingPacket.o \ node/InetAddress.o \ -- cgit v1.2.3 From 02d18af57d7d05d26e44ff2015f5bcf55ebce7a2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 23 Jun 2017 16:10:26 -0700 Subject: Remove Cluster.hpp --- node/IncomingPacket.cpp | 1 - node/Network.cpp | 1 - node/Node.cpp | 1 - node/Peer.cpp | 1 - node/Switch.cpp | 1 - 5 files changed, 5 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 9140c502..1d55c9f3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -42,7 +42,6 @@ #include "Salsa20.hpp" #include "SHA512.hpp" #include "World.hpp" -#include "Cluster.hpp" #include "Node.hpp" #include "CertificateOfMembership.hpp" #include "CertificateOfRepresentation.hpp" diff --git a/node/Network.cpp b/node/Network.cpp index 74d81941..12deeeb7 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -42,7 +42,6 @@ #include "NetworkController.hpp" #include "Node.hpp" #include "Peer.hpp" -#include "Cluster.hpp" // Uncomment to make the rules engine dump trace info to stdout //#define ZT_RULES_ENGINE_DEBUGGING 1 diff --git a/node/Node.cpp b/node/Node.cpp index 37586834..39325b65 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -45,7 +45,6 @@ #include "Address.hpp" #include "Identity.hpp" #include "SelfAwareness.hpp" -#include "Cluster.hpp" #include "Network.hpp" const struct sockaddr_storage ZT_SOCKADDR_NULL = {0}; diff --git a/node/Peer.cpp b/node/Peer.cpp index 01905833..84086048 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -32,7 +32,6 @@ #include "Switch.hpp" #include "Network.hpp" #include "SelfAwareness.hpp" -#include "Cluster.hpp" #include "Packet.hpp" namespace ZeroTier { diff --git a/node/Switch.cpp b/node/Switch.cpp index 211b706a..2be54b37 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -43,7 +43,6 @@ #include "Peer.hpp" #include "SelfAwareness.hpp" #include "Packet.hpp" -#include "Cluster.hpp" namespace ZeroTier { -- cgit v1.2.3 From 355cce3938a815feba1085569263ae0225cebfa6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 27 Jun 2017 11:31:29 -0700 Subject: Rename Utils::snprintf due to it being a #define on one platform. --- attic/DBM.cpp | 243 ------------------------------- attic/DBM.hpp | 168 --------------------- controller/EmbeddedNetworkController.cpp | 26 ++-- controller/JSONDB.cpp | 10 +- node/Address.hpp | 4 +- node/Dictionary.hpp | 4 +- node/InetAddress.cpp | 8 +- node/MAC.hpp | 2 +- node/MulticastGroup.hpp | 2 +- node/Network.cpp | 4 +- node/NetworkConfig.cpp | 2 +- node/Node.cpp | 2 +- node/Utils.cpp | 5 +- node/Utils.hpp | 3 +- one.cpp | 20 +-- osdep/BSDEthernetTap.cpp | 16 +- osdep/Http.cpp | 4 +- osdep/LinuxEthernetTap.cpp | 10 +- osdep/OSUtils.cpp | 6 +- osdep/OSXEthernetTap.cpp | 14 +- osdep/PortMapper.cpp | 4 +- osdep/WindowsEthernetTap.cpp | 10 +- selftest.cpp | 2 +- service/ClusterDefinition.hpp | 2 +- service/OneService.cpp | 62 ++++---- service/SoftwareUpdater.cpp | 2 +- 26 files changed, 111 insertions(+), 524 deletions(-) delete mode 100644 attic/DBM.cpp delete mode 100644 attic/DBM.hpp (limited to 'node') diff --git a/attic/DBM.cpp b/attic/DBM.cpp deleted file mode 100644 index 54f017e0..00000000 --- a/attic/DBM.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#include "DBM.hpp" - -#include "../version.h" - -#include "../node/Salsa20.hpp" -#include "../node/Poly1305.hpp" -#include "../node/SHA512.hpp" - -#include "../osdep/OSUtils.hpp" - -#define ZT_STORED_OBJECT_TYPE__CLUSTER_NODE_STATUS (ZT_STORED_OBJECT__MAX_TYPE_ID + 1) -#define ZT_STORED_OBJECT_TYPE__CLUSTER_DEFINITION (ZT_STORED_OBJECT__MAX_TYPE_ID + 2) - -namespace ZeroTier { - -// We generate the cluster ID from our address and version info since this is -// not at all designed to allow interoperation between versions (or endians) -// in the same cluster. -static inline uint64_t _mkClusterId(const Address &myAddress) -{ - uint64_t x = ZEROTIER_ONE_VERSION_MAJOR; - x <<= 8; - x += ZEROTIER_ONE_VERSION_MINOR; - x <<= 8; - x += ZEROTIER_ONE_VERSION_REVISION; - x <<= 40; - x ^= myAddress.toInt(); -#if __BYTE_ORDER == __BIG_ENDIAN - ++x; -#endif; - return x; -} - -void DBM::onUpdate(uint64_t from,const _MapKey &k,const _MapValue &v,uint64_t rev) -{ - char p[4096]; - char tmp[ZT_DBM_MAX_VALUE_SIZE]; - if (_persistentPath((ZT_StoredObjectType)k.type,k.key,p,sizeof(p))) { - // Reduce unnecessary disk writes - FILE *f = fopen(p,"r"); - if (f) { - long n = (long)fread(tmp,1,sizeof(tmp),f); - fclose(f); - if ((n == (long)v.len)&&(!memcmp(v.data,tmp,n))) - return; - } - - // Write to disk if file has changed or was not already present - f = fopen(p,"w"); - if (f) { - if (fwrite(data,len,1,f) != 1) - fprintf(stderr,"WARNING: error writing to %s (I/O error)" ZT_EOL_S,p); - fclose(f); - if (type == ZT_STORED_OBJECT_IDENTITY_SECRET) - OSUtils::lockDownFile(p,false); - } else { - fprintf(stderr,"WARNING: error writing to %s (cannot open)" ZT_EOL_S,p); - } - } -} - -void DBM::onDelete(uint64_t from,const _MapKey &k) -{ - char p[4096]; - if (_persistentPath((ZT_StoredObjectType)k.type,k.key,p,sizeof(p))) - OSUtils::rm(p); -} - -DBM::_vsdm_cryptor::_vsdm_cryptor(const Identity &secretIdentity) -{ - uint8_t s512[64]; - SHA512::hash(h512,secretIdentity.privateKeyPair().priv.data,ZT_C25519_PRIVATE_KEY_LEN); - memcpy(_key,s512,sizeof(_key)); -} - -void DBM::_vsdm_cryptor::encrypt(void *d,unsigned long l) -{ - if (l >= 24) { // sanity check - uint8_t key[32]; - uint8_t authKey[32]; - uint8_t auth[16]; - - uint8_t *const iv = reinterpret_cast(d) + (l - 16); - Utils::getSecureRandom(iv,16); - memcpy(key,_key,32); - for(unsigned long i=0;i<8;++i) - _key[i] ^= iv[i]; - - Salsa20 s20(key,iv + 8); - memset(authKey,0,32); - s20.crypt12(authKey,authKey,32); - s20.crypt12(d,d,l - 24); - - Poly1305::compute(auth,d,l - 24,authKey); - memcpy(reinterpret_cast(d) + (l - 24),auth,8); - } -} - -bool DBM::_vsdm_cryptor::decrypt(void *d,unsigned long l) -{ - if (l >= 24) { // sanity check - uint8_t key[32]; - uint8_t authKey[32]; - uint8_t auth[16]; - - uint8_t *const iv = reinterpret_cast(d) + (l - 16); - memcpy(key,_key,32); - for(unsigned long i=0;i<8;++i) - _key[i] ^= iv[i]; - - Salsa20 s20(key,iv + 8); - memset(authKey,0,32); - s20.crypt12(authKey,authKey,32); - - Poly1305::compute(auth,d,l - 24,authKey); - if (!Utils::secureEq(reinterpret_cast(d) + (l - 24),auth,8)) - return false; - - s20.crypt12(d,d,l - 24); - - return true; - } - return false; -} - -DBM::DBM(const Identity &secretIdentity,uint64_t clusterMemberId,const std::string &basePath,Node *node) : - _basePath(basePath), - _node(node), - _startTime(OSUtils::now()), - _m(_mkClusterId(secretIdentity.address()),clusterMemberId,false,_vsdm_cryptor(secretIdentity),_vsdm_watcher(this)) -{ -} - -DBM::~DBM() -{ -} - -void DBM::put(const ZT_StoredObjectType type,const uint64_t key,const void *data,unsigned int len) -{ - char p[4096]; - if (_m.put(_MapKey(key,(uint16_t)type),Value(OSUtils::now(),(uint16_t)len,data))) { - if (_persistentPath(type,key,p,sizeof(p))) { - FILE *f = fopen(p,"w"); - if (f) { - if (fwrite(data,len,1,f) != 1) - fprintf(stderr,"WARNING: error writing to %s (I/O error)" ZT_EOL_S,p); - fclose(f); - if (type == ZT_STORED_OBJECT_IDENTITY_SECRET) - OSUtils::lockDownFile(p,false); - } else { - fprintf(stderr,"WARNING: error writing to %s (cannot open)" ZT_EOL_S,p); - } - } - } -} - -bool DBM::get(const ZT_StoredObjectType type,const uint64_t key,Value &value) -{ - char p[4096]; - if (_m.get(_MapKey(key,(uint16_t)type),value)) - return true; - if (_persistentPath(type,key,p,sizeof(p))) { - FILE *f = fopen(p,"r"); - if (f) { - long n = (long)fread(value.data,1,sizeof(value.data),f); - value.len = (n > 0) ? (uint16_t)n : (uint16_t)0; - fclose(f); - value.ts = OSUtils::getLastModified(p); - _m.put(_MapKey(key,(uint16_t)type),value); - return true; - } - } - return false; -} - -void DBM::del(const ZT_StoredObjectType type,const uint64_t key) -{ - char p[4096]; - _m.del(_MapKey(key,(uint16_t)type)); - if (_persistentPath(type,key,p,sizeof(p))) - OSUtils::rm(p); -} - -void DBM::clean() -{ -} - -bool DBM::_persistentPath(const ZT_StoredObjectType type,const uint64_t key,char *p,unsigned int maxlen) -{ - switch(type) { - case ZT_STORED_OBJECT_IDENTITY_PUBLIC: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "identity.public",_basePath.c_str()); - return true; - case ZT_STORED_OBJECT_IDENTITY_SECRET: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "identity.secret",_basePath.c_str()); - return true; - case ZT_STORED_OBJECT_IDENTITY: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "iddb.d" ZT_PATH_SEPARATOR_S "%.10llx",_basePath.c_str(),key); - return true; - case ZT_STORED_OBJECT_NETWORK_CONFIG: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf",_basePath.c_str(),key); - return true; - case ZT_STORED_OBJECT_PLANET: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "planet",_basePath.c_str()); - return true; - case ZT_STORED_OBJECT_MOON: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "moons.d" ZT_PATH_SEPARATOR_S "%.16llx.moon",_basePath.c_str(),key); - return true; - case (ZT_StoredObjectType)ZT_STORED_OBJECT_TYPE__CLUSTER_DEFINITION: - Utils::snprintf(p,maxlen,"%s" ZT_PATH_SEPARATOR_S "cluster",_basePath.c_str()); - return true; - default: - return false; - } -} - -} // namespace ZeroTier diff --git a/attic/DBM.hpp b/attic/DBM.hpp deleted file mode 100644 index c6d5b8c0..00000000 --- a/attic/DBM.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_DBM_HPP___ -#define ZT_DBM_HPP___ - -#include -#include -#include -#include - -#include - -#include "../node/Constants.hpp" -#include "../node/Mutex.hpp" -#include "../node/Utils.hpp" -#include "../node/Identity.hpp" -#include "../node/Peer.hpp" - -#include "../ext/vsdm/vsdm.hpp" - -// The Peer is the largest structure we persist here -#define ZT_DBM_MAX_VALUE_SIZE sizeof(Peer) - -namespace ZeroTier { - -class Node; -class DBM; - -class DBM -{ -public: - ZT_PACKED_STRUCT(struct Value - { - Value(const uint64_t t,const uint16_t l,const void *d) : - ts(t), - l(l) - { - memcpy(data,d,l); - } - uint64_t ts; - uint16_t len; - uint8_t data[ZT_DBM_MAX_VALUE_SIZE]; - }); - -private: - ZT_PACKED_STRUCT(struct _MapKey - { - _MapKey() : obj(0),type(0) {} - _MapKey(const uint16_t t,const uint64_t o) : obj(o),type(t) {} - uint64_t obj; - uint16_t type; - inline bool operator==(const _MapKey &k) const { return ((obj == k.obj)&&(type == k.type)); } - }); - struct _MapHasher - { - inline std::size_t operator()(const _MapKey &k) const { return (std::size_t)((k.obj ^ (k.obj >> 32)) + (uint64_t)k.type); } - }; - - void onUpdate(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev); - void onDelete(uint64_t from,const _MapKey &k); - - class _vsdm_watcher - { - public: - _vsdm_watcher(DBM *p) : _parent(p) {} - inline void add(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev) { _parent->onUpdate(from,k,v,rev); } - inline void update(uint64_t from,const _MapKey &k,const Value &v,uint64_t rev) { _parent->onUpdate(from,k,v,rev); } - inline void del(uint64_t from,const _MapKey &k) { _parent->onDelete(from,k); } - private: - DBM *_parent; - }; - class _vsdm_serializer - { - public: - static inline unsigned long objectSize(const _MapKey &k) { return 10; } - static inline unsigned long objectSize(const Value &v) { return (10 + v.len); } - static inline const char *objectData(const _MapKey &k) { return reinterpret_cast(&k); } - static inline const char *objectData(const Value &v) { return reinterpret_cast(&v); } - static inline bool objectDeserialize(const char *d,unsigned long l,_MapKey &k) - { - if (l == 10) { - memcpy(&k,d,10); - return true; - } - return false; - } - static inline bool objectDeserialize(const char *d,unsigned long l,Value &v) - { - if ((l >= 10)&&(l <= (10 + ZT_DBM_MAX_VALUE_SIZE))) { - memcpy(&v,d,l); - return true; - } - return false; - } - }; - class _vsdm_cryptor - { - public: - _vsdm_cryptor(const Identity &secretIdentity); - static inline unsigned long overhead() { return 24; } - void encrypt(void *d,unsigned long l); - bool decrypt(void *d,unsigned long l); - uint8_t _key[32]; - }; - - typedef vsdm< _MapKey,Value,16384,_vsdm_watcher,_vsdm_serializer,_vsdm_cryptor,_MapHasher > _Map; - - friend class _Map; - -public: - ZT_PACKED_STRUCT(struct ClusterPeerStatus - { - uint64_t startTime; - uint64_t currentTime; - uint64_t clusterPeersConnected; - uint64_t ztPeersConnected; - uint16_t platform; - uint16_t arch; - }); - - DBM(const Identity &secretIdentity,uint64_t clusterMemberId,const std::string &basePath,Node *node); - - ~DBM(); - - void put(const ZT_StoredObjectType type,const uint64_t key,const void *data,unsigned int len); - - bool get(const ZT_StoredObjectType type,const uint64_t key,Value &value); - - void del(const ZT_StoredObjectType type,const uint64_t key); - - void clean(); - -private: - bool DBM::_persistentPath(const ZT_StoredObjectType type,const uint64_t key,char *p,unsigned int maxlen); - - const std::string _basePath; - Node *const _node; - uint64_t _startTime; - _Map _m; -}; - -} // namespace ZeroTier - -#endif diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index e2eaa788..85c759e7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -122,12 +122,12 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: @@ -179,7 +179,7 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); r["mask"] = tmp; break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: @@ -514,7 +514,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { char tmp[128]; - Utils::snprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); responseBody.append(tmp); } }); @@ -548,7 +548,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { if (responseBody.length() > 1) responseBody.push_back(','); - Utils::snprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); + Utils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); responseBody.append(tmp); } responseBody.push_back(']'); @@ -562,7 +562,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( // Controller status char tmp[4096]; - Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + Utils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); responseBody = tmp; responseContentType = "application/json"; return 200; @@ -603,14 +603,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + Utils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member; _db.getNetworkMember(nwid,address,member); @@ -748,7 +748,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!nwid) return 503; } - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); json network; _db.getNetwork(nwid,network); @@ -995,7 +995,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _queue.post(qe); char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); + Utils::ztsnprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); responseBody = tmp; responseContentType = "application/json"; @@ -1083,7 +1083,7 @@ void EmbeddedNetworkController::threadMain() auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) lrt = ms->second.lastRequestTime; - Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + Utils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, @@ -1093,7 +1093,7 @@ void EmbeddedNetworkController::threadMain() }); } char tmp2[256]; - Utils::snprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + Utils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); pong.append(tmp2); _db.writeRaw("pong",pong); } @@ -1126,7 +1126,7 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index e0dd1742..acf23700 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -94,7 +94,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) std::string body; std::map reqHeaders; char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); reqHeaders["Content-Length"] = tmp; reqHeaders["Content-Type"] = "application/json"; const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); @@ -164,7 +164,7 @@ bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlo void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { char n[64]; - Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig)); { Mutex::Lock _l(_networks_m); @@ -176,7 +176,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) { char n[256]; - Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); writeRaw(n,OSUtils::jsonDump(memberConfig)); { Mutex::Lock _l(_networks_m); @@ -202,7 +202,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) } char n[256]; - Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); if (_httpAddr) { // Deletion is currently done by Central in harnessed mode @@ -229,7 +229,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) { char n[256]; - Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); if (_httpAddr) { // Deletion is currently done by the caller in Central harnessed mode diff --git a/node/Address.hpp b/node/Address.hpp index 9d2d1734..98e32858 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -144,7 +144,7 @@ public: inline std::string toString() const { char buf[16]; - Utils::snprintf(buf,sizeof(buf),"%.10llx",(unsigned long long)_a); + Utils::ztsnprintf(buf,sizeof(buf),"%.10llx",(unsigned long long)_a); return std::string(buf); }; @@ -154,7 +154,7 @@ public: */ inline void toString(char *buf,unsigned int len) const { - Utils::snprintf(buf,len,"%.10llx",(unsigned long long)_a); + Utils::ztsnprintf(buf,len,"%.10llx",(unsigned long long)_a); } /** diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 4413d628..0b000f13 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -391,7 +391,7 @@ public: inline bool add(const char *key,uint64_t value) { char tmp[32]; - Utils::snprintf(tmp,sizeof(tmp),"%llx",(unsigned long long)value); + Utils::ztsnprintf(tmp,sizeof(tmp),"%llx",(unsigned long long)value); return this->add(key,tmp,-1); } @@ -401,7 +401,7 @@ public: inline bool add(const char *key,const Address &a) { char tmp[32]; - Utils::snprintf(tmp,sizeof(tmp),"%.10llx",(unsigned long long)a.toInt()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",(unsigned long long)a.toInt()); return this->add(key,tmp,-1); } diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 0fbb2d68..17d7c72e 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -152,7 +152,7 @@ std::string InetAddress::toString() const char buf[128]; switch(ss_family) { case AF_INET: - Utils::snprintf(buf,sizeof(buf),"%d.%d.%d.%d/%d", + Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d/%d", (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], @@ -161,7 +161,7 @@ std::string InetAddress::toString() const ); return std::string(buf); case AF_INET6: - Utils::snprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d", + Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d", (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), @@ -190,7 +190,7 @@ std::string InetAddress::toIpString() const char buf[128]; switch(ss_family) { case AF_INET: - Utils::snprintf(buf,sizeof(buf),"%d.%d.%d.%d", + Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d", (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], @@ -198,7 +198,7 @@ std::string InetAddress::toIpString() const ); return std::string(buf); case AF_INET6: - Utils::snprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", + Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), diff --git a/node/MAC.hpp b/node/MAC.hpp index e7717d99..db50aeb1 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -178,7 +178,7 @@ public: */ inline void toString(char *buf,unsigned int len) const { - Utils::snprintf(buf,len,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)(*this)[0],(int)(*this)[1],(int)(*this)[2],(int)(*this)[3],(int)(*this)[4],(int)(*this)[5]); + Utils::ztsnprintf(buf,len,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)(*this)[0],(int)(*this)[1],(int)(*this)[2],(int)(*this)[3],(int)(*this)[4],(int)(*this)[5]); } /** diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index 4240db67..7cbec2e0 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -100,7 +100,7 @@ public: inline std::string toString() const { char buf[64]; - Utils::snprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi); + Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi); return std::string(buf); } diff --git a/node/Network.cpp b/node/Network.cpp index 12deeeb7..8c6f2ce8 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -51,7 +51,7 @@ namespace ZeroTier { namespace { #ifdef ZT_RULES_ENGINE_DEBUGGING -#define FILTER_TRACE(f,...) { Utils::snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } +#define FILTER_TRACE(f,...) { Utils::ztsnprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) { switch(rt) { @@ -1261,7 +1261,7 @@ void Network::requestConfiguration(void *tPtr) nconf->rules[13].t = (uint8_t)ZT_NETWORK_RULE_ACTION_DROP; nconf->type = ZT_NETWORK_TYPE_PUBLIC; - Utils::snprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); + Utils::ztsnprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); this->setConfiguration(tPtr,*nconf,false); delete nconf; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index c39f6cab..65101c3a 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -94,7 +94,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (ets.length() > 0) ets.push_back(','); char tmp2[16]; - Utils::snprintf(tmp2,sizeof(tmp2),"%x",et); + Utils::ztsnprintf(tmp2,sizeof(tmp2),"%x",et); ets.append(tmp2); } et = 0; diff --git a/node/Node.cpp b/node/Node.cpp index 39325b65..ab49e63b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -742,7 +742,7 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) va_end(ap); tmp2[sizeof(tmp2)-1] = (char)0; - Utils::snprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); + Utils::ztsnprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); postEvent((void *)0,ZT_EVENT_TRACE,tmp1); } #endif // ZT_TRACE diff --git a/node/Utils.cpp b/node/Utils.cpp index d69e5335..d2321e16 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -244,8 +244,7 @@ bool Utils::scopy(char *dest,unsigned int len,const char *src) return true; } -unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...) - throw(std::length_error) +unsigned int Utils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...) { va_list ap; @@ -256,7 +255,7 @@ unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...) if ((n >= (int)len)||(n < 0)) { if (len) buf[len - 1] = (char)0; - throw std::length_error("buf[] overflow in Utils::snprintf"); + throw std::length_error("buf[] overflow"); } return (unsigned int)n; diff --git a/node/Utils.hpp b/node/Utils.hpp index 25a90055..212ef247 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -244,8 +244,7 @@ public: * @param ... Format arguments * @throws std::length_error buf[] too short (buf[] will still be left null-terminated) */ - static unsigned int snprintf(char *buf,unsigned int len,const char *fmt,...) - throw(std::length_error); + static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...); /** * Count the number of bits set in an integer diff --git a/one.cpp b/one.cpp index 93504cfb..cbf09121 100644 --- a/one.cpp +++ b/one.cpp @@ -260,9 +260,9 @@ static int cli(int argc,char **argv) if (hd) { char p[4096]; #ifdef __APPLE__ - Utils::snprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); + Utils::ztsnprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); #else - Utils::snprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); + Utils::ztsnprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); #endif OSUtils::readFile(p,authToken); } @@ -278,7 +278,7 @@ static int cli(int argc,char **argv) InetAddress addr; { char addrtmp[256]; - Utils::snprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); + Utils::ztsnprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); addr = InetAddress(addrtmp); } @@ -366,7 +366,7 @@ static int cli(int argc,char **argv) std::string addr = path["address"]; const uint64_t now = OSUtils::now(); const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; - Utils::snprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); bestPath = tmp; break; } @@ -378,7 +378,7 @@ static int cli(int argc,char **argv) int64_t vmin = p["versionMinor"]; int64_t vrev = p["versionRev"]; if (vmaj >= 0) { - Utils::snprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + Utils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); } else { ver[0] = '-'; ver[1] = (char)0; @@ -527,9 +527,9 @@ static int cli(int argc,char **argv) const uint64_t seed = Utils::hexStrToU64(arg2.c_str()); if ((worldId)&&(seed)) { char jsons[1024]; - Utils::snprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); + Utils::ztsnprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); char cl[128]; - Utils::snprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -579,11 +579,11 @@ static int cli(int argc,char **argv) if (eqidx != std::string::npos) { if ((arg2.substr(0,eqidx) == "allowManaged")||(arg2.substr(0,eqidx) == "allowGlobal")||(arg2.substr(0,eqidx) == "allowDefault")) { char jsons[1024]; - Utils::snprintf(jsons,sizeof(jsons),"{\"%s\":%s}", + Utils::ztsnprintf(jsons,sizeof(jsons),"{\"%s\":%s}", arg2.substr(0,eqidx).c_str(), (((arg2.substr(eqidx,2) == "=t")||(arg2.substr(eqidx,2) == "=1")) ? "true" : "false")); char cl[128]; - Utils::snprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -864,7 +864,7 @@ static int idtool(int argc,char **argv) Buffer wbuf; w.serialize(wbuf); char fn[128]; - Utils::snprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); + Utils::ztsnprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); OSUtils::writeFile(fn,wbuf.data(),wbuf.size()); printf("wrote %s (signed world with timestamp %llu)" ZT_EOL_S,fn,(unsigned long long)now); } diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index 5bb5fbd1..f07f9e5a 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -114,8 +114,8 @@ BSDEthernetTap::BSDEthernetTap( std::vector devFiles(OSUtils::listDirectory("/dev")); for(int i=9993;i<(9993+128);++i) { - Utils::snprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::snprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); if (std::find(devFiles.begin(),devFiles.end(),std::string(tmpdevname)) == devFiles.end()) { long cpid = (long)vfork(); if (cpid == 0) { @@ -152,8 +152,8 @@ BSDEthernetTap::BSDEthernetTap( /* Other BSDs like OpenBSD only have a limited number of tap devices that cannot be renamed */ for(int i=0;i<64;++i) { - Utils::snprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::snprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { _dev = tmpdevname; @@ -171,9 +171,9 @@ BSDEthernetTap::BSDEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::snprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); + Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -385,7 +385,7 @@ void BSDEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%u",mtu); + Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/Http.cpp b/osdep/Http.cpp index f1d3bfe2..3c556f44 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -244,10 +244,10 @@ unsigned int Http::_do( try { char tmp[1024]; - Utils::snprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); handler.writeBuf.append(tmp); for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) { - Utils::snprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); handler.writeBuf.append(tmp); } handler.writeBuf.append("\r\n"); diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index ccaa92ef..fc5199f1 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -97,7 +97,7 @@ LinuxEthernetTap::LinuxEthernetTap( char procpath[128],nwids[32]; struct stat sbuf; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally @@ -134,7 +134,7 @@ LinuxEthernetTap::LinuxEthernetTap( std::map::const_iterator gdmEntry = globalDeviceMap.find(nwids); if (gdmEntry != globalDeviceMap.end()) { Utils::scopy(ifr.ifr_name,sizeof(ifr.ifr_name),gdmEntry->second.c_str()); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); recalledDevice = (stat(procpath,&sbuf) != 0); } @@ -142,8 +142,8 @@ LinuxEthernetTap::LinuxEthernetTap( #ifdef __SYNOLOGY__ int devno = 50; do { - Utils::snprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + Utils::ztsnprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); + Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); // try zt#++ until we find one that does not exist #else char devno = 0; @@ -158,7 +158,7 @@ LinuxEthernetTap::LinuxEthernetTap( _base32_5_to_8(reinterpret_cast(tmp2) + 5,tmp3 + 10); tmp3[15] = (char)0; memcpy(ifr.ifr_name,tmp3,16); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); #endif } diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 53e8bb97..bf0dc04d 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -134,7 +134,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) if (date.QuadPart > 0) { date.QuadPart -= adjust.QuadPart; if ((uint64_t)((date.QuadPart / 10000000) * 1000) < olderThan) { - Utils::snprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); + Utils::ztsnprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); if (DeleteFileA(tmp)) ++cleaned; } @@ -157,7 +157,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) break; if (dptr) { if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_REG)) { - Utils::snprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); + Utils::ztsnprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); if (stat(tmp,&st) == 0) { uint64_t mt = (uint64_t)(st.st_mtime); if ((mt > 0)&&((mt * 1000) < olderThan)) { @@ -446,7 +446,7 @@ std::string OSUtils::jsonString(const nlohmann::json &jv,const char *dfl) return jv; } else if (jv.is_number()) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + Utils::ztsnprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); return tmp; } else if (jv.is_boolean()) { return ((bool)jv ? std::string("1") : std::string("0")); diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index f5e1c43f..e082408e 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -336,7 +336,7 @@ OSXEthernetTap::OSXEthernetTap( char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; struct stat stattmp; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); + Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _gl(globalTapCreateLock); @@ -391,13 +391,13 @@ OSXEthernetTap::OSXEthernetTap( // Open the first unused tap device if we didn't recall a previous one. if (!recalledDevice) { for(int i=0;i<64;++i) { - Utils::snprintf(devpath,sizeof(devpath),"/dev/zt%d",i); + Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); if (stat(devpath,&stattmp)) throw std::runtime_error("no more TAP devices available"); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { char foo[16]; - Utils::snprintf(foo,sizeof(foo),"zt%d",i); + Utils::ztsnprintf(foo,sizeof(foo),"zt%d",i); _dev = foo; break; } @@ -413,9 +413,9 @@ OSXEthernetTap::OSXEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::snprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::snprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::snprintf(metstr,sizeof(metstr),"%u",_metric); + Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -636,7 +636,7 @@ void OSXEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%u",mtu); + Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index 99286172..df868e7a 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -205,7 +205,7 @@ public: memset(externalip,0,sizeof(externalip)); memset(&urls,0,sizeof(urls)); memset(&data,0,sizeof(data)); - Utils::snprintf(inport,sizeof(inport),"%d",localPort); + Utils::ztsnprintf(inport,sizeof(inport),"%d",localPort); if ((UPNP_GetValidIGD(devlist,&urls,&data,lanaddr,sizeof(lanaddr)))&&(lanaddr[0])) { #ifdef ZT_PORTMAPPER_TRACE @@ -220,7 +220,7 @@ public: int tryPort = (int)localPort + tries; if (tryPort >= 65535) tryPort = (tryPort - 65535) + 1025; - Utils::snprintf(outport,sizeof(outport),"%u",tryPort); + Utils::ztsnprintf(outport,sizeof(outport),"%u",tryPort); // First check and see if this port is already mapped to the // same unique name. If so, keep this mapping and don't try diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index c5d82d8e..b96ad791 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -484,7 +484,7 @@ WindowsEthernetTap::WindowsEthernetTap( char tag[24]; // We "tag" registry entries with the network ID to identify persistent devices - Utils::snprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); + Utils::ztsnprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); Mutex::Lock _l(_systemTapInitLock); @@ -601,10 +601,10 @@ WindowsEthernetTap::WindowsEthernetTap( if (_netCfgInstanceId.length() > 0) { char tmps[64]; - unsigned int tmpsl = Utils::snprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; + unsigned int tmpsl = Utils::ztsnprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"NetworkAddress",REG_SZ,tmps,tmpsl); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MAC",REG_SZ,tmps,tmpsl); - tmpsl = Utils::snprintf(tmps, sizeof(tmps), "%d", mtu); + tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MTU",REG_SZ,tmps,tmpsl); DWORD tmp = 0; @@ -879,7 +879,7 @@ void WindowsEthernetTap::setMtu(unsigned int mtu) HKEY nwAdapters; if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\Class\\{4D36E972-E325-11CE-BFC1-08002BE10318}", 0, KEY_READ | KEY_WRITE, &nwAdapters) == ERROR_SUCCESS) { char tmps[64]; - unsigned int tmpsl = Utils::snprintf(tmps, sizeof(tmps), "%d", mtu); + unsigned int tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters, _mySubkeyName.c_str(), "MTU", REG_SZ, tmps, tmpsl); RegCloseKey(nwAdapters); } @@ -902,7 +902,7 @@ void WindowsEthernetTap::threadMain() HANDLE wait4[3]; OVERLAPPED tapOvlRead,tapOvlWrite; - Utils::snprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); + Utils::ztsnprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); try { while (_run) { diff --git a/selftest.cpp b/selftest.cpp index 8175d708..ff171aa3 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -831,7 +831,7 @@ static int testOther() memset(key, 0, sizeof(key)); memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - Utils::snprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); + Utils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); int r = rand() % 128; for(int x=0;x lines(OSUtils::split(cf.c_str(),"\r\n","","")); for(std::vector::iterator l(lines.begin());l!=lines.end();++l) { diff --git a/service/OneService.cpp b/service/OneService.cpp index 644454bc..993fb116 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -210,10 +210,10 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc, case ZT_NETWORK_TYPE_PUBLIC: ntype = "PUBLIC"; break; } - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); nj["id"] = tmp; nj["nwid"] = tmp; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); nj["mac"] = tmp; nj["name"] = nc->name; nj["status"] = nstatus; @@ -260,12 +260,12 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break; } - Utils::snprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); pj["address"] = tmp; pj["versionMajor"] = peer->versionMajor; pj["versionMinor"] = peer->versionMinor; pj["versionRev"] = peer->versionRev; - Utils::snprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); + Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); pj["version"] = tmp; pj["latency"] = peer->latency; pj["role"] = prole; @@ -289,7 +289,7 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) static void _moonToJson(nlohmann::json &mj,const World &world) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",world.id()); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",world.id()); mj["id"] = tmp; mj["timestamp"] = world.timestamp(); mj["signature"] = Utils::hex(world.signature().data,(unsigned int)world.signature().size()); @@ -687,7 +687,7 @@ public: // Save primary port to a file so CLIs and GUIs can learn it easily char portstr[64]; - Utils::snprintf(portstr,sizeof(portstr),"%u",_ports[0]); + Utils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr)); // Attempt to bind to a secondary port chosen from our ZeroTier address. @@ -725,7 +725,7 @@ public: } if (_ports[2]) { char uniqueName[64]; - Utils::snprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); + Utils::ztsnprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); _portMapper = new PortMapper(_ports[2],uniqueName); } } @@ -1069,7 +1069,7 @@ public: n->second.settings = settings; char nlcpath[4096]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); + Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); FILE *out = fopen(nlcpath,"w"); if (out) { fprintf(out,"allowManaged=%d\n",(int)n->second.settings.allowManaged); @@ -1188,7 +1188,7 @@ public: ZT_NodeStatus status; _node->status(&status); - Utils::snprintf(tmp,sizeof(tmp),"%.10llx",status.address); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",status.address); res["address"] = tmp; res["publicIdentity"] = status.publicIdentity; res["online"] = (bool)(status.online != 0); @@ -1197,7 +1197,7 @@ public: res["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; res["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; res["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; - Utils::snprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); + Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); res["version"] = tmp; res["clock"] = OSUtils::now(); @@ -1373,7 +1373,7 @@ public: if ((scode != 200)&&(seed != 0)) { char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",id); + Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",id); res["id"] = tmp; res["roots"] = json::array(); res["timestamp"] = 0; @@ -1617,7 +1617,7 @@ public: std::string h = controllerDbHttpHost; _controllerDbPath.append(h); char dbp[128]; - Utils::snprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); + Utils::ztsnprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort); _controllerDbPath.push_back(':'); _controllerDbPath.append(dbp); if (controllerDbHttpPath.is_string()) { @@ -1711,7 +1711,7 @@ public: if (syncRoutes) { char tapdev[64]; #ifdef __WINDOWS__ - Utils::snprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); + Utils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); #else Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str()); #endif @@ -1933,24 +1933,24 @@ public: bool secure = false; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); secure = true; break; case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); secure = true; break; case ZT_STATE_OBJECT_PLANET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; default: p[0] = (char)0; @@ -2022,7 +2022,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -2235,7 +2235,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -2402,7 +2402,7 @@ public: if (!n.tap) { try { char friendlyName[128]; - Utils::snprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); + Utils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); n.tap = new EthernetTap( _homePath.c_str(), @@ -2416,7 +2416,7 @@ public: *nuptr = (void *)&n; char nlcpath[256]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); std::string nlcbuf; if (OSUtils::readFile(nlcpath,nlcbuf)) { Dictionary<4096> nc; @@ -2502,7 +2502,7 @@ public: #endif if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) { char nlcpath[256]; - Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); OSUtils::rm(nlcpath); } } else { @@ -2554,22 +2554,22 @@ public: char p[4096]; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); break; case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_PLANET: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::snprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; default: return -1; @@ -2765,7 +2765,7 @@ public: default: scodestr = "Error"; break; } - Utils::snprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + Utils::ztsnprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", scode, scodestr, contentType.c_str(), diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index d94beab5..e0519827 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -284,7 +284,7 @@ bool SoftwareUpdater::check(const uint64_t now) if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; char tmp[512]; - const unsigned int len = Utils::snprintf(tmp,sizeof(tmp), + const unsigned int len = Utils::ztsnprintf(tmp,sizeof(tmp), "%c{\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "\":%d," -- cgit v1.2.3 From baa10c2995b7e0e49b49fe63a264a20982b817cf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Jun 2017 17:32:07 -0700 Subject: . --- include/ZeroTierOne.h | 8 +-- node/Constants.hpp | 2 +- node/IncomingPacket.cpp | 8 +-- node/Network.cpp | 14 +++-- node/Node.cpp | 24 ++++--- node/Node.hpp | 6 +- node/Path.hpp | 30 +++++++++ node/Peer.cpp | 164 ++++++++++++++++++++++++++++++++++++++++++------ node/Peer.hpp | 54 ++++++---------- node/Topology.cpp | 32 +++++++--- node/Topology.hpp | 2 +- service/OneService.cpp | 130 +++++++++++++++++++++----------------- 12 files changed, 326 insertions(+), 148 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 9c295cee..40cae3b4 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1115,14 +1115,14 @@ enum ZT_StateObjectType * Canonical path: /peers.d/
(10-digit hex address) * Persistence: optional, can be purged at any time */ - ZT_STATE_OBJECT_PEER = 3, + ZT_STATE_OBJECT_PEER_STATE = 3, /** * The identity of a known peer * * Object ID: peer address * Canonical path: /iddb.d/
(10-digit hex address) - * Persistence: optional, can be purged at any time, recommended ttl 30-60 days + * Persistence: recommended, can be purged at any time, recommended ttl 30-60 days */ ZT_STATE_OBJECT_PEER_IDENTITY = 4, @@ -1248,7 +1248,7 @@ typedef void (*ZT_StatePutFunction)( void *, /* User ptr */ void *, /* Thread ptr */ enum ZT_StateObjectType, /* State object type */ - uint64_t, /* State object ID (if applicable) */ + const uint64_t [2], /* State object ID (if applicable) */ const void *, /* State object data */ int); /* Length of data or -1 to delete */ @@ -1264,7 +1264,7 @@ typedef int (*ZT_StateGetFunction)( void *, /* User ptr */ void *, /* Thread ptr */ enum ZT_StateObjectType, /* State object type */ - uint64_t, /* State object ID (if applicable) */ + const uint64_t [2], /* State object ID (if applicable) */ void *, /* Buffer to store state object data */ unsigned int); /* Length of data buffer in bytes */ diff --git a/node/Constants.hpp b/node/Constants.hpp index fbbba76e..88549937 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -216,7 +216,7 @@ /** * How often Topology::clean() and Network::clean() and similar are called, in ms */ -#define ZT_HOUSEKEEPING_PERIOD 120000 +#define ZT_HOUSEKEEPING_PERIOD 10000 /** * How long to remember peer records in RAM if they haven't been used diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 1d55c9f3..4d99e87d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1211,8 +1211,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path { - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) - peer->setClusterPreferred(a); + //if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) + // peer->setClusterPreferred(a); if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); @@ -1228,8 +1228,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path { - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) - peer->setClusterPreferred(a); + //if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) + // peer->setClusterPreferred(a); if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); diff --git a/node/Network.cpp b/node/Network.cpp index 8c6f2ce8..0a16ded8 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -700,10 +700,13 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u this->setConfiguration(tPtr,*nconf,false); _lastConfigUpdate = 0; // still want to re-request since it's likely outdated } else { + uint64_t tmp[2]; + tmp[0] = nwid; tmp[1] = 0; + bool got = false; Dictionary *dict = new Dictionary(); try { - int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1); + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,dict->unsafeData(),ZT_NETWORKCONFIG_DICT_CAPACITY - 1); if (n > 1) { NetworkConfig *nconf = new NetworkConfig(); try { @@ -719,7 +722,7 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u delete dict; if (!got) - RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,"\n",1); + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,"\n",1); } if (!_portInitialized) { @@ -1185,8 +1188,11 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD if (saveToDisk) { Dictionary *d = new Dictionary(); try { - if (nconf.toDictionary(*d,false)) - RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,_id,d->data(),d->sizeBytes()); + if (nconf.toDictionary(*d,false)) { + uint64_t tmp[2]; + tmp[0] = _id; tmp[1] = 0; + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,d->data(),d->sizeBytes()); + } } catch ( ... ) {} delete d; } diff --git a/node/Node.cpp b/node/Node.cpp index ab49e63b..1112c0f2 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -76,22 +76,26 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); + uint64_t idtmp[2]; + idtmp[0] = 0; idtmp[1] = 0; char tmp[512]; std::string tmp2; - int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,0,tmp,sizeof(tmp) - 1); + int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; if (!RR->identity.fromString(tmp)) n = -1; } + + idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; if (n <= 0) { RR->identity.generate(); tmp2 = RR->identity.toString(true); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length()); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp2.data(),(unsigned int)tmp2.length()); tmp2 = RR->identity.toString(false); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length()); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length()); } else { - n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp,sizeof(tmp) - 1); + n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; if (RR->identity.toString(false) != tmp) @@ -99,7 +103,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 } if (n <= 0) { tmp2 = RR->identity.toString(false); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,RR->identity.address().toInt(),tmp2.data(),(unsigned int)tmp2.length()); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length()); } } @@ -145,7 +149,7 @@ ZT_ResultCode Node::processStateUpdate( ZT_ResultCode r = ZT_RESULT_OK_IGNORED; switch(type) { - case ZT_STATE_OBJECT_PEER: + case ZT_STATE_OBJECT_PEER_STATE: if (len) { } break; @@ -380,9 +384,9 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint } if ((now - _lastHousekeepingRun) >= ZT_HOUSEKEEPING_PERIOD) { + _lastHousekeepingRun = now; try { - _lastHousekeepingRun = now; - RR->topology->clean(now); + RR->topology->doPeriodicTasks(tptr,now); RR->sa->clean(now); RR->mc->clean(now); } catch ( ... ) { @@ -443,7 +447,9 @@ ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) _networks.erase(nwid); } - RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,nwid); + uint64_t tmp[2]; + tmp[0] = nwid; tmp[1] = 0; + RR->node->stateObjectDelete(tptr,ZT_STATE_OBJECT_NETWORK_CONFIG,tmp); return ZT_RESULT_OK; } diff --git a/node/Node.hpp b/node/Node.hpp index f407c60c..f1209d00 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -197,9 +197,9 @@ public: inline bool online() const throw() { return _online; } - inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id,void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,maxlen); } - inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,(int)len); } - inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,(const void *)0,-1); } + inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,maxlen); } + inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,(int)len); } + inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2]) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,(const void *)0,-1); } #ifdef ZT_TRACE void postTrace(const char *module,unsigned int line,const char *fmt,...); diff --git a/node/Path.hpp b/node/Path.hpp index 32bceae0..74b31d8d 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -46,6 +46,11 @@ */ #define ZT_PATH_MAX_PREFERENCE_RANK ((ZT_INETADDRESS_MAX_SCOPE << 1) | 1) +/** + * Maximum distance for a path + */ +#define ZT_PATH_DISTANCE_MAX 0xffff + namespace ZeroTier { class RuntimeEnvironment; @@ -120,6 +125,7 @@ public: _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), + _distance(ZT_PATH_DISTANCE_MAX), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -137,6 +143,7 @@ public: _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), + _distance(ZT_PATH_DISTANCE_MAX), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -299,6 +306,28 @@ public: */ inline uint64_t lastIn() const { return _lastIn; } + /** + * @return Time last trust-established packet was received + */ + inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } + + /** + * @return Distance (higher is further) + */ + inline unsigned int distance() const { return _distance; } + + /** + * @param lo Last out send + * @param li Last in send + * @param lt Last trust established packet received + */ + inline void updateFromRemoteState(const uint64_t lo,const uint64_t li,const uint64_t lt) + { + _lastOut = lo; + _lastIn = li; + _lastTrustEstablishedPacketReceived = lt; + } + /** * Return and increment outgoing packet counter (used with Packet::armor()) * @@ -315,6 +344,7 @@ private: volatile signed int _incomingLinkQualitySlowLogCounter; volatile unsigned int _incomingLinkQualityPreviousPacketCounter; volatile unsigned int _outgoingPacketCounter; + volatile unsigned int _distance; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often diff --git a/node/Peer.cpp b/node/Peer.cpp index 84086048..a7466296 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,6 +38,8 @@ namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : RR(renv), + _lastWroteState(0), + _lastReceivedStateTimestamp(0), _lastReceive(0), _lastNontrivialReceive(0), _lastTriedMemorizedPath(0), @@ -75,6 +77,7 @@ void Peer::received( { const uint64_t now = RR->node->now(); +/* #ifdef ZT_ENABLE_CLUSTER bool isClusterSuboptimalPath = false; if ((RR->cluster)&&(hops == 0)) { @@ -120,6 +123,7 @@ void Peer::received( } } #endif +*/ _lastReceive = now; switch (verb) { @@ -143,6 +147,8 @@ void Peer::received( if (hops == 0) { bool pathAlreadyKnown = false; + bool newPathLearned = false; + { Mutex::Lock _l(_paths_m); if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) { @@ -152,9 +158,6 @@ void Peer::received( const struct sockaddr_in *const ll = reinterpret_cast(&(_v4Path.p->localAddress())); if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(rl->sin_addr.s_addr == ll->sin_addr.s_addr)&&(rl->sin_port == ll->sin_port)) { _v4Path.lr = now; -#ifdef ZT_ENABLE_CLUSTER - _v4Path.localClusterSuboptimal = isClusterSuboptimalPath; -#endif pathAlreadyKnown = true; } } else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) { @@ -164,9 +167,6 @@ void Peer::received( const struct sockaddr_in6 *const ll = reinterpret_cast(&(_v6Path.p->localAddress())); if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(!memcmp(rl->sin6_addr.s6_addr,ll->sin6_addr.s6_addr,16))&&(rl->sin6_port == ll->sin6_port)) { _v6Path.lr = now; -#ifdef ZT_ENABLE_CLUSTER - _v6Path.localClusterSuboptimal = isClusterSuboptimalPath; -#endif pathAlreadyKnown = true; } } @@ -176,11 +176,11 @@ void Peer::received( Mutex::Lock _l(_paths_m); _PeerPath *potentialNewPeerPath = (_PeerPath *)0; if (path->address().ss_family == AF_INET) { - if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) { + if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) { potentialNewPeerPath = &_v4Path; } } else if (path->address().ss_family == AF_INET6) { - if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) { + if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) { potentialNewPeerPath = &_v6Path; } } @@ -188,11 +188,7 @@ void Peer::received( if (verb == Packet::VERB_OK) { potentialNewPeerPath->lr = now; potentialNewPeerPath->p = path; -#ifdef ZT_ENABLE_CLUSTER - potentialNewPeerPath->localClusterSuboptimal = isClusterSuboptimalPath; - if (RR->cluster) - RR->cluster->broadcastHavePeer(_id); -#endif + newPathLearned = true; } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); @@ -200,15 +196,12 @@ void Peer::received( } } } + + if (newPathLearned) + writeState(tPtr,now); } else if (this->trustEstablished(now)) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) -#ifdef ZT_ENABLE_CLUSTER - // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection - const bool haveCluster = (RR->cluster); -#else - const bool haveCluster = false; -#endif - if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) { + if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) { _lastDirectPathPushSent = now; std::vector pathsToPush; @@ -439,4 +432,135 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } +void Peer::writeState(void *tPtr,const uint64_t now) +{ + try { + Buffer b; + + b.append((uint8_t)1); // version + b.append(now); + + _id.serialize(b); + + { + Mutex::Lock _l(_paths_m); + unsigned int count = 0; + if (_v4Path.lr) + ++count; + if (_v6Path.lr) + ++count; + b.append((uint8_t)count); + if (_v4Path.lr) { + b.append(_v4Path.lr); + b.append(_v4Path.p->lastOut()); + b.append(_v4Path.p->lastIn()); + b.append(_v4Path.p->lastTrustEstablishedPacketReceived()); + b.append((uint16_t)_v4Path.p->distance()); + _v4Path.p->address().serialize(b); + _v4Path.p->localAddress().serialize(b); + } + if (_v6Path.lr) { + b.append(_v6Path.lr); + b.append(_v6Path.p->lastOut()); + b.append(_v6Path.p->lastIn()); + b.append(_v6Path.p->lastTrustEstablishedPacketReceived()); + b.append((uint16_t)_v6Path.p->distance()); + _v6Path.p->address().serialize(b); + _v6Path.p->localAddress().serialize(b); + } + } + + b.append(_lastReceive); + b.append(_lastNontrivialReceive); + b.append(_lastTriedMemorizedPath); + b.append(_lastDirectPathPushSent); + b.append(_lastDirectPathPushReceive); + b.append(_lastCredentialRequestSent); + b.append(_lastWhoisRequestReceived); + b.append(_lastEchoRequestReceived); + b.append(_lastComRequestReceived); + b.append(_lastComRequestSent); + b.append(_lastCredentialsReceived); + b.append(_lastTrustEstablishedPacketReceived); + + b.append(_vProto); + b.append(_vMajor); + b.append(_vMinor); + b.append(_vRevision); + + b.append((uint16_t)0); // length of additional fields + + uint64_t tmp[2]; + tmp[0] = _id.address().toInt(); tmp[1] = 0; + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size()); + + _lastWroteState = now; + } catch ( ... ) {} // sanity check, should not be possible +} + +bool Peer::applyStateUpdate(const void *data,unsigned int len) +{ + try { + Buffer b(data,len); + unsigned int ptr = 0; + + if (b[ptr++] != 1) + return false; + const uint64_t ts = b.at(ptr); ptr += 8; + if (ts <= _lastReceivedStateTimestamp) + return false; + + const unsigned int pathCount = (unsigned int)b[ptr++]; + { + Mutex::Lock _l(_paths_m); + for(unsigned int i=0;i(ptr); ptr += 8; + const uint64_t lastOut = b.at(ptr); ptr += 8; + const uint64_t lastIn = b.at(ptr); ptr += 8; + const uint64_t lastTrustEstablishedPacketReceived = b.at(ptr); ptr += 8; + const unsigned int distance = b.at(ptr); ptr += 2; + InetAddress addr,localAddr; + ptr += addr.deserialize(b,ptr); + ptr += localAddr.deserialize(b,ptr); + if (addr.ss_family == localAddr.ss_family) { + _PeerPath *p = (_PeerPath *)0; + switch(addr.ss_family) { + case AF_INET: p = &_v4Path; break; + case AF_INET6: p = &_v6Path; break; + } + if (p) { + if ( ((p->p->address() != addr)||(p->p->localAddress() != localAddr)) && (p->p->distance() > distance) ) + p->p = RR->topology->getPath(localAddr,addr); + p->lr = lr; + p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); + } + } + } + } + + _lastReceive = std::max(_lastReceive,b.at(ptr)); ptr += 8; + _lastNontrivialReceive = std::max(_lastNontrivialReceive,b.at(ptr)); ptr += 8; + _lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,b.at(ptr)); ptr += 8; + _lastDirectPathPushSent = std::max(_lastDirectPathPushSent,b.at(ptr)); ptr += 8; + _lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,b.at(ptr)); ptr += 8; + _lastCredentialRequestSent = std::max(_lastCredentialRequestSent,b.at(ptr)); ptr += 8; + _lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,b.at(ptr)); ptr += 8; + _lastEchoRequestReceived = std::max(_lastEchoRequestReceived,b.at(ptr)); ptr += 8; + _lastComRequestReceived = std::max(_lastComRequestReceived,b.at(ptr)); ptr += 8; + _lastComRequestSent = std::max(_lastComRequestSent,b.at(ptr)); ptr += 8; + _lastCredentialsReceived = std::max(_lastCredentialsReceived,b.at(ptr)); ptr += 8; + _lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,b.at(ptr)); ptr += 8; + + _vProto = b.at(ptr); ptr += 2; + _vMajor = b.at(ptr); ptr += 2; + _vMinor = b.at(ptr); ptr += 2; + _vRevision = b.at(ptr); ptr += 2; + + _lastReceivedStateTimestamp = ts; + + return true; + } catch ( ... ) {} // ignore invalid state updates + return false; +} + } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 9b57f23e..d6b7dad9 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -193,6 +193,22 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); + /** + * Write current peer state to external storage / cluster network + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + */ + void writeState(void *tPtr,const uint64_t now); + + /** + * Apply a state update received from e.g. a remote cluster member + * + * @param data State update data + * @param len Length of state update + * @return True if state update was applied, false if ignored or invalid + */ + bool applyStateUpdate(const void *data,unsigned int len); + /** * Reset paths within a given IP scope and address family * @@ -218,19 +234,6 @@ public: } } - /** - * Indicate that the given address was provided by a cluster as a preferred destination - * - * @param addr Address cluster prefers that we use - */ - inline void setClusterPreferred(const InetAddress &addr) - { - if (addr.ss_family == AF_INET) - _v4ClusterPreferred = addr; - else if (addr.ss_family == AF_INET6) - _v6ClusterPreferred = addr; - } - /** * Fill parameters with V4 and V6 addresses if known and alive * @@ -317,18 +320,6 @@ public: else _latency = std::min(l,(unsigned int)65535); } -#ifdef ZT_ENABLE_CLUSTER - /** - * @param now Current time - * @return True if this peer has at least one active direct path that is not cluster-suboptimal - */ - inline bool hasLocalClusterOptimalPath(uint64_t now) const - { - Mutex::Lock _l(_paths_m); - return ( ((_v4Path.p)&&(_v4Path.p->alive(now))&&(!_v4Path.localClusterSuboptimal)) || ((_v6Path.p)&&(_v6Path.p->alive(now))&&(!_v6Path.localClusterSuboptimal)) ); - } -#endif - /** * @return 256-bit secret symmetric encryption key */ @@ -449,22 +440,18 @@ public: private: struct _PeerPath { -#ifdef ZT_ENABLE_CLUSTER - _PeerPath() : lr(0),p(),localClusterSuboptimal(false) {} -#else _PeerPath() : lr(0),p() {} -#endif uint64_t lr; // time of last valid ZeroTier packet SharedPtr p; -#ifdef ZT_ENABLE_CLUSTER - bool localClusterSuboptimal; // true if our cluster has determined that we should not be serving this peer -#endif }; uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; const RuntimeEnvironment *RR; + uint64_t _lastWroteState; + uint64_t _lastReceivedStateTimestamp; + uint64_t _lastReceive; // direct or indirect uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. uint64_t _lastTriedMemorizedPath; @@ -483,9 +470,6 @@ private: uint16_t _vMinor; uint16_t _vRevision; - InetAddress _v4ClusterPreferred; - InetAddress _v6ClusterPreferred; - _PeerPath _v4Path; // IPv4 direct path _PeerPath _v6Path; // IPv6 direct path Mutex _paths_m; diff --git a/node/Topology.cpp b/node/Topology.cpp index d4b424ff..be116b28 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -69,7 +69,9 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : _amRoot(false) { uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH]; - int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,0,tmp,sizeof(tmp)); + uint64_t idtmp[2]; + idtmp[0] = 0; idtmp[1] = 0; + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PLANET,idtmp,tmp,sizeof(tmp)); if (n > 0) { try { World cachedPlanet; @@ -159,7 +161,9 @@ void Topology::saveIdentity(void *tPtr,const Identity &id) { if (id) { const std::string tmp(id.toString(false)); - RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,id.address().toInt(),tmp.data(),(unsigned int)tmp.length()); + uint64_t idtmp[2]; + idtmp[0] = id.address().toInt(); idtmp[1] = 0; + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp.data(),(unsigned int)tmp.length()); } } @@ -329,7 +333,9 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew) try { Buffer sbuf; existing->serialize(sbuf,false); - RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,existing->id(),sbuf.data(),sbuf.size()); + uint64_t idtmp[2]; + idtmp[0] = existing->id(); idtmp[1] = 0; + RR->node->stateObjectPut(tPtr,(existing->type() == World::TYPE_PLANET) ? ZT_STATE_OBJECT_PLANET : ZT_STATE_OBJECT_MOON,idtmp,sbuf.data(),sbuf.size()); } catch ( ... ) {} _memoizeUpstreams(tPtr); @@ -340,7 +346,9 @@ bool Topology::addWorld(void *tPtr,const World &newWorld,bool alwaysAcceptNew) void Topology::addMoon(void *tPtr,const uint64_t id,const Address &seed) { char tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH]; - int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,id,tmp,sizeof(tmp)); + uint64_t idtmp[2]; + idtmp[0] = id; idtmp[1] = 0; + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_MOON,idtmp,tmp,sizeof(tmp)); if (n > 0) { try { World w; @@ -369,7 +377,9 @@ void Topology::removeMoon(void *tPtr,const uint64_t id) if (m->id() != id) { nm.push_back(*m); } else { - RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,id); + uint64_t idtmp[2]; + idtmp[0] = id; idtmp[1] = 0; + RR->node->stateObjectDelete(tPtr,ZT_STATE_OBJECT_MOON,idtmp); } } _moons.swap(nm); @@ -384,7 +394,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id) _memoizeUpstreams(tPtr); } -void Topology::clean(uint64_t now) +void Topology::doPeriodicTasks(void *tPtr,uint64_t now) { { Mutex::Lock _l1(_peers_m); @@ -393,10 +403,14 @@ void Topology::clean(uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) + if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) { _peers.erase(*a); + } else { + (*p)->writeState(tPtr,now); + } } } + { Mutex::Lock _l(_paths_m); Hashtable< Path::HashKey,SharedPtr >::Iterator i(_paths); @@ -412,7 +426,9 @@ void Topology::clean(uint64_t now) Identity Topology::_getIdentity(void *tPtr,const Address &zta) { char tmp[512]; - int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,zta.toInt(),tmp,sizeof(tmp) - 1); + uint64_t idtmp[2]; + idtmp[0] = zta.toInt(); idtmp[1] = 0; + int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; try { diff --git a/node/Topology.hpp b/node/Topology.hpp index d06ba94b..9bc7c0d8 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -308,7 +308,7 @@ public: /** * Clean and flush database */ - void clean(uint64_t now); + void doPeriodicTasks(void *tPtr,uint64_t now); /** * @param now Current time diff --git a/service/OneService.cpp b/service/OneService.cpp index 993fb116..f949f348 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -312,8 +312,8 @@ class OneServiceImpl; static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf); static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData); -static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,const void *data,int len); -static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen); +static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len); +static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen); static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); @@ -1220,34 +1220,20 @@ public: res["planetWorldId"] = planet.id(); res["planetWorldTimestamp"] = planet.timestamp(); -/* -#ifdef ZT_ENABLE_CLUSTER - json cj; - ZT_ClusterStatus cs; - _node->clusterStatus(&cs); - if (cs.clusterSize >= 1) { - json cja = json::array(); - for(unsigned int i=0;i::const_iterator ca(_clusterBackplaneAddresses.begin());ca!=_clusterBackplaneAddresses.end();++ca) { + uint64_t up = 0; + for(std::vector::const_iterator c(_tcpConnections.begin());c!=_tcpConnections.end();++c) { + if (((*c)->remoteAddr == *ca)&&((*c)->clusterMemberId)&&((*c)->lastReceive > up)) + up = (*c)->lastReceive; + } + cj[ca->toString()] = up; } - cj["members"] = cja; - cj["myId"] = (int)cs.myId; - cj["clusterSize"] = cs.clusterSize; + res["cluster"] = cj; } - res["cluster"] = cj; -#else - res["cluster"] = json(); -#endif -*/ scode = 200; } else if (ps[0] == "moon") { @@ -1877,16 +1863,15 @@ public: return false; } - void replicateStateObject(const ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len,TcpConnection *tc) + void replicateStateObject(const ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len,TcpConnection *tc) { - char buf[34]; - + char buf[42]; Mutex::Lock _l2(tc->writeq_m); if (tc->writeq.length() == 0) _phy.setNotifyWritable(tc->sock,true); - const unsigned int mlen = len + 34; + const unsigned int mlen = len + 42; tc->writeq.push_back((char)((mlen >> 16) & 0xff)); tc->writeq.push_back((char)((mlen >> 8) & 0xff)); @@ -1895,24 +1880,32 @@ public: Utils::getSecureRandom(buf,16); buf[24] = (char)CLUSTER_MESSAGE_STATE_OBJECT; buf[25] = (char)type; - buf[26] = (char)((id >> 56) & 0xff); - buf[27] = (char)((id >> 48) & 0xff); - buf[28] = (char)((id >> 40) & 0xff); - buf[29] = (char)((id >> 32) & 0xff); - buf[30] = (char)((id >> 24) & 0xff); - buf[31] = (char)((id >> 16) & 0xff); - buf[32] = (char)((id >> 8) & 0xff); - buf[33] = (char)(id & 0xff); + buf[26] = (char)((id[0] >> 56) & 0xff); + buf[27] = (char)((id[0] >> 48) & 0xff); + buf[28] = (char)((id[0] >> 40) & 0xff); + buf[29] = (char)((id[0] >> 32) & 0xff); + buf[30] = (char)((id[0] >> 24) & 0xff); + buf[31] = (char)((id[0] >> 16) & 0xff); + buf[32] = (char)((id[0] >> 8) & 0xff); + buf[33] = (char)(id[0] & 0xff); + buf[34] = (char)((id[1] >> 56) & 0xff); + buf[35] = (char)((id[1] >> 48) & 0xff); + buf[36] = (char)((id[1] >> 40) & 0xff); + buf[37] = (char)((id[1] >> 32) & 0xff); + buf[38] = (char)((id[1] >> 24) & 0xff); + buf[39] = (char)((id[1] >> 16) & 0xff); + buf[40] = (char)((id[1] >> 8) & 0xff); + buf[41] = (char)(id[1] & 0xff); const unsigned long startpos = (unsigned long)tc->writeq.length(); - tc->writeq.append(buf,34); + tc->writeq.append(buf,42); tc->writeq.append(reinterpret_cast(data),len); char *const outdata = const_cast(tc->writeq.data()) + startpos; encryptClusterMessage(outdata,mlen); } - void replicateStateObjectToCluster(const ZT_StateObjectType type,const uint64_t id,const void *const data,const unsigned int len,const uint64_t everyoneBut) + void replicateStateObjectToCluster(const ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len,const uint64_t everyoneBut) { std::vector sentTo; if (everyoneBut) @@ -1927,7 +1920,7 @@ public: } } - void writeStateObject(enum ZT_StateObjectType type,uint64_t id,const void *data,int len) + void writeStateObject(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { char p[4096]; bool secure = false; @@ -1940,17 +1933,17 @@ public: secure = true; break; case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id[0]); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); secure = true; break; case ZT_STATE_OBJECT_PLANET: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); break; default: p[0] = (char)0; @@ -1985,8 +1978,12 @@ public: if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + *f).c_str(),buf)) { if (f->length() == 21) { const uint64_t nwid = Utils::hexStrToU64(f->substr(0,16).c_str()); - if (nwid) - replicateStateObject(ZT_STATE_OBJECT_NETWORK_CONFIG,nwid,buf.data(),(int)buf.length(),tc); + if (nwid) { + uint64_t tmp[2]; + tmp[0] = nwid; + tmp[1] = 0; + replicateStateObject(ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,buf.data(),(int)buf.length(),tc); + } } } } @@ -1996,8 +1993,12 @@ public: if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + *f).c_str(),buf)) { if (f->length() == 21) { const uint64_t moonId = Utils::hexStrToU64(f->substr(0,16).c_str()); - if (moonId) - replicateStateObject(ZT_STATE_OBJECT_MOON,moonId,buf.data(),(int)buf.length(),tc); + if (moonId) { + uint64_t tmp[2]; + tmp[0] = moonId; + tmp[1] = 0; + replicateStateObject(ZT_STATE_OBJECT_MOON,tmp,buf.data(),(int)buf.length(),tc); + } } } } @@ -2313,8 +2314,9 @@ public: break; case CLUSTER_MESSAGE_STATE_OBJECT: - if (mlen >= (25 + 9)) { // type + object ID + [data] - const uint64_t objId = ( + if (mlen >= 42) { // type + object ID + [data] + uint64_t objId[2]; + objId[0] = ( ((uint64_t)data[26] << 56) | ((uint64_t)data[27] << 48) | ((uint64_t)data[28] << 40) | @@ -2324,9 +2326,19 @@ public: ((uint64_t)data[32] << 8) | (uint64_t)data[33] ); - if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34)) == ZT_RESULT_OK) { - writeStateObject((ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34)); - replicateStateObjectToCluster((ZT_StateObjectType)data[25],objId,data + 34,(unsigned int)(mlen - 34),tc->clusterMemberId); + objId[1] = ( + ((uint64_t)data[34] << 56) | + ((uint64_t)data[35] << 48) | + ((uint64_t)data[36] << 40) | + ((uint64_t)data[37] << 32) | + ((uint64_t)data[38] << 24) | + ((uint64_t)data[39] << 16) | + ((uint64_t)data[40] << 8) | + (uint64_t)data[41] + ); + if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId[0],data + 42,(unsigned int)(mlen - 42)) == ZT_RESULT_OK) { + writeStateObject((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42)); + replicateStateObjectToCluster((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42),tc->clusterMemberId); } } break; @@ -2543,13 +2555,13 @@ public: } } - inline void nodeStatePutFunction(enum ZT_StateObjectType type,uint64_t id,const void *data,int len) + inline void nodeStatePutFunction(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { writeStateObject(type,id,data,len); replicateStateObjectToCluster(type,id,data,len,0); } - inline int nodeStateGetFunction(enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen) + inline int nodeStateGetFunction(enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen) { char p[4096]; switch(type) { @@ -2866,9 +2878,9 @@ static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr { return reinterpret_cast(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); } static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData) { reinterpret_cast(uptr)->nodeEventCallback(event,metaData); } -static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,const void *data,int len) +static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { reinterpret_cast(uptr)->nodeStatePutFunction(type,id,data,len); } -static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,uint64_t id,void *data,unsigned int maxlen) +static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen) { return reinterpret_cast(uptr)->nodeStateGetFunction(type,id,data,maxlen); } static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) { return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); } -- cgit v1.2.3 From 2f20258807f8665bc3f9c527106e61761e01ecc3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 10:25:36 -0700 Subject: . --- include/ZeroTierOne.h | 34 +++++----- node/Constants.hpp | 12 ++-- node/Identity.hpp | 5 +- node/IncomingPacket.cpp | 12 ---- node/Network.cpp | 10 --- node/Node.cpp | 151 +++++++++++--------------------------------- node/Node.hpp | 2 +- node/Path.hpp | 13 ---- node/Peer.cpp | 113 ++++++++++++++++++++------------- node/Peer.hpp | 16 ++++- node/RuntimeEnvironment.hpp | 19 ++++-- node/Switch.cpp | 93 ++------------------------- node/Topology.cpp | 59 +++++------------ node/Topology.hpp | 27 ++------ osdep/Binder.hpp | 4 +- service/OneService.cpp | 94 +++++++++++++-------------- 16 files changed, 240 insertions(+), 424 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 40cae3b4..133ae340 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1072,9 +1072,9 @@ typedef struct * identity of a node and its address, the identity (public and secret) * must be saved at a minimum. * - * The reference service implementation currently persists identity, - * peer identities (for a period of time), planet, moons, and network - * configurations. Other state is treated as ephemeral. + * State objects actually have two IDs (uint64_t[2]). If only one is + * listed the second ([1]) should be zero and is ignored in storage + * and replication. * * All state objects should be replicated in cluster mode. The reference * clustering implementation uses a rumor mill algorithm in which state @@ -1118,22 +1118,25 @@ enum ZT_StateObjectType ZT_STATE_OBJECT_PEER_STATE = 3, /** - * The identity of a known peer + * Network configuration * * Object ID: peer address - * Canonical path: /iddb.d/
(10-digit hex address) - * Persistence: recommended, can be purged at any time, recommended ttl 30-60 days + * Canonical path: /networks.d/.conf (16-digit hex ID) + * Persistence: required if network memberships should persist */ - ZT_STATE_OBJECT_PEER_IDENTITY = 4, + ZT_STATE_OBJECT_NETWORK_CONFIG = 4, /** - * Network configuration + * Network membership (network X peer intersection) * - * Object ID: peer address - * Canonical path: /networks.d/.conf (16-digit hex ID) - * Persistence: required if network memberships should persist + * If these are persisted they must be restored after peer states and + * network configs. Otherwise they are ignored. + * + * Object ID: [0] network ID, [1] peer address + * Canonical path: /networks.d//members.d/
+ * Persistence: optional (not usually needed) */ - ZT_STATE_OBJECT_NETWORK_CONFIG = 5, + ZT_STATE_OBJECT_NETWORK_MEMBERSHIP = 5, /** * The planet (there is only one per... well... planet!) @@ -1450,7 +1453,8 @@ void ZT_Node_delete(ZT_Node *node); * * Unless clustering is being implemented this function doesn't need to be * used after startup. It could be called in response to filesystem changes - * to allow some degree of live configurability by filesystem observation. + * to allow some degree of live configurability by filesystem observation + * but this kind of thing is entirely optional. * * The return value of this function indicates whether the update was accepted * as new. A return value of ZT_RESULT_OK indicates that the node gleaned new @@ -1468,7 +1472,7 @@ void ZT_Node_delete(ZT_Node *node); * @param node Node instance * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param type State object type - * @param id State object ID + * @param id State object ID (if object type has only one ID, second should be zero) * @param data State object data * @param len Length of state object data in bytes * @return ZT_RESULT_OK if object was accepted or ZT_RESULT_OK_IGNORED if non-informative, error if object was invalid @@ -1477,7 +1481,7 @@ enum ZT_ResultCode ZT_Node_processStateUpdate( ZT_Node *node, void *tptr, ZT_StateObjectType type, - uint64_t id, + const uint64_t id[2], const void *data, unsigned int len); diff --git a/node/Constants.hpp b/node/Constants.hpp index 88549937..274b9564 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -216,7 +216,12 @@ /** * How often Topology::clean() and Network::clean() and similar are called, in ms */ -#define ZT_HOUSEKEEPING_PERIOD 10000 +#define ZT_HOUSEKEEPING_PERIOD 60000 + +/** + * How often in ms to write peer state to storage and/or cluster (approximate) + */ +#define ZT_PEER_STATE_WRITE_PERIOD 10000 /** * How long to remember peer records in RAM if they haven't been used @@ -322,11 +327,6 @@ */ #define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) -/** - * Send a full HELLO every this often (ms) - */ -#define ZT_PEER_SEND_FULL_HELLO_EVERY (ZT_PEER_PING_PERIOD * 2) - /** * How often to retry expired paths that we're still remembering */ diff --git a/node/Identity.hpp b/node/Identity.hpp index b1c7d6f4..79e17f4d 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -91,7 +91,10 @@ public: ~Identity() { - delete _privateKey; + if (_privateKey) { + Utils::burn(_privateKey,sizeof(C25519::Private)); + delete _privateKey; + } } inline Identity &operator=(const Identity &id) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 4d99e87d..0548387b 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -585,12 +585,6 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar } else { // Request unknown WHOIS from upstream from us (if we have one) RR->sw->requestWhois(tPtr,addr); -#ifdef ZT_ENABLE_CLUSTER - // Distribute WHOIS queries across a cluster if we do not know the ID. - // This may result in duplicate OKs to the querying peer, which is fine. - if (RR->cluster) - RR->cluster->sendDistributedQuery(*this); -#endif } } @@ -1055,12 +1049,6 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - - // If we are a member of a cluster, distribute this GATHER across it -#ifdef ZT_ENABLE_CLUSTER - if ((RR->cluster)&&(gatheredLocally < gatherLimit)) - RR->cluster->sendDistributedQuery(*this); -#endif } peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); diff --git a/node/Network.cpp b/node/Network.cpp index 0a16ded8..bccc0397 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1067,11 +1067,6 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add return 0; } -#ifdef ZT_ENABLE_CLUSTER - if ((source)&&(RR->cluster)) - RR->cluster->broadcastNetworkConfigChunk(chunk.field(start,chunk.size() - start),chunk.size() - start); -#endif - // New properly verified chunks can be flooded "virally" through the network if (fastPropagate) { Address *a = (Address *)0; @@ -1099,11 +1094,6 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add if ((!c)||(_incomingConfigChunks[i].ts < c->ts)) c = &(_incomingConfigChunks[i]); } - -#ifdef ZT_ENABLE_CLUSTER - if ((source)&&(RR->cluster)) - RR->cluster->broadcastNetworkConfigChunk(chunk.field(start,chunk.size() - start),chunk.size() - start); -#endif } else { TRACE("discarded single-chunk unsigned legacy config: this is only allowed if the sender is the controller itself"); return 0; diff --git a/node/Node.cpp b/node/Node.cpp index 1112c0f2..4ffe496c 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -68,6 +68,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 throw std::runtime_error("callbacks struct version mismatch"); memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); + // Initialize non-cryptographic PRNG from a good random source Utils::getSecureRandom((void *)_prngState,sizeof(_prngState)); _online = false; @@ -78,33 +79,34 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 uint64_t idtmp[2]; idtmp[0] = 0; idtmp[1] = 0; - char tmp[512]; - std::string tmp2; + char tmp[1024]; int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; - if (!RR->identity.fromString(tmp)) + if (RR->identity.fromString(tmp)) { + RR->publicIdentityStr = RR->identity.toString(false); + RR->secretIdentityStr = RR->identity.toString(true); + } else { n = -1; + } } idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; if (n <= 0) { RR->identity.generate(); - tmp2 = RR->identity.toString(true); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp2.data(),(unsigned int)tmp2.length()); - tmp2 = RR->identity.toString(false); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length()); + RR->publicIdentityStr = RR->identity.toString(false); + RR->secretIdentityStr = RR->identity.toString(true); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr.data(),(unsigned int)RR->secretIdentityStr.length()); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr.data(),(unsigned int)RR->publicIdentityStr.length()); } else { n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; - if (RR->identity.toString(false) != tmp) + if (RR->publicIdentityStr != tmp) n = -1; } - if (n <= 0) { - tmp2 = RR->identity.toString(false); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp2.data(),(unsigned int)tmp2.length()); - } + if (n <= 0) + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr.data(),(unsigned int)RR->publicIdentityStr.length()); } try { @@ -125,24 +127,20 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 Node::~Node() { - Mutex::Lock _l(_networks_m); - - _networks.clear(); // destroy all networks before shutdown - + { + Mutex::Lock _l(_networks_m); + _networks.clear(); // destroy all networks before shutdown + } delete RR->sa; delete RR->topology; delete RR->mc; delete RR->sw; - -#ifdef ZT_ENABLE_CLUSTER - delete RR->cluster; -#endif } ZT_ResultCode Node::processStateUpdate( void *tptr, ZT_StateObjectType type, - uint64_t id, + const uint64_t id[2], const void *data, unsigned int len) { @@ -151,11 +149,12 @@ ZT_ResultCode Node::processStateUpdate( case ZT_STATE_OBJECT_PEER_STATE: if (len) { - } - break; - - case ZT_STATE_OBJECT_PEER_IDENTITY: - if (len) { + const SharedPtr p(RR->topology->getPeer(tptr,Address(id[0]))); + if (p) { + r = (p->applyStateUpdate(data,len)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED; + } else { + r = (Peer::createFromStateUpdate(RR,tptr,data,len)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED; + } } break; @@ -163,9 +162,9 @@ ZT_ResultCode Node::processStateUpdate( if (len <= (ZT_NETWORKCONFIG_DICT_CAPACITY - 1)) { if (len < 2) { Mutex::Lock _l(_networks_m); - SharedPtr &nw = _networks[id]; + SharedPtr &nw = _networks[id[0]]; if (!nw) { - nw = SharedPtr(new Network(RR,tptr,id,(void *)0,(const NetworkConfig *)0)); + nw = SharedPtr(new Network(RR,tptr,id[0],(void *)0,(const NetworkConfig *)0)); r = ZT_RESULT_OK; } } else { @@ -175,7 +174,7 @@ ZT_ResultCode Node::processStateUpdate( try { if (nconf->fromDictionary(*dict)) { Mutex::Lock _l(_networks_m); - SharedPtr &nw = _networks[id]; + SharedPtr &nw = _networks[id[0]]; if (nw) { switch (nw->setConfiguration(tptr,*nconf,false)) { default: @@ -189,7 +188,7 @@ ZT_ResultCode Node::processStateUpdate( break; } } else { - nw = SharedPtr(new Network(RR,tptr,id,(void *)0,nconf)); + nw = SharedPtr(new Network(RR,tptr,id[0],(void *)0,nconf)); } } else { r = ZT_RESULT_ERROR_BAD_PARAMETER; @@ -208,9 +207,14 @@ ZT_ResultCode Node::processStateUpdate( } break; + case ZT_STATE_OBJECT_NETWORK_MEMBERSHIP: + if (len) { + } + break; + case ZT_STATE_OBJECT_PLANET: case ZT_STATE_OBJECT_MOON: - if (len <= ZT_WORLD_MAX_SERIALIZED_LENGTH) { + if ((len)&&(len <= ZT_WORLD_MAX_SERIALIZED_LENGTH)) { World w; try { w.deserialize(Buffer(data,len)); @@ -395,18 +399,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint } try { -#ifdef ZT_ENABLE_CLUSTER - // If clustering is enabled we have to call cluster->doPeriodicTasks() very often, so we override normal timer deadline behavior - if (RR->cluster) { - RR->sw->doTimerTasks(tptr,now); - RR->cluster->doPeriodicTasks(); - *nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate - } else { -#endif - *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); -#ifdef ZT_ENABLE_CLUSTER - } -#endif + *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -620,76 +613,6 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } -/* -ZT_ResultCode Node::clusterInit( - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg) -{ -#ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - return ZT_RESULT_ERROR_BAD_PARAMETER; - - std::vector eps; - for(unsigned int i=0;icluster = new Cluster(RR,myId,eps,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg); - - return ZT_RESULT_OK; -#else - return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION; -#endif -} - -ZT_ResultCode Node::clusterAddMember(unsigned int memberId) -{ -#ifdef ZT_ENABLE_CLUSTER - if (!RR->cluster) - return ZT_RESULT_ERROR_BAD_PARAMETER; - RR->cluster->addMember((uint16_t)memberId); - return ZT_RESULT_OK; -#else - return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION; -#endif -} - -void Node::clusterRemoveMember(unsigned int memberId) -{ -#ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - RR->cluster->removeMember((uint16_t)memberId); -#endif -} - -void Node::clusterHandleIncomingMessage(const void *msg,unsigned int len) -{ -#ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - RR->cluster->handleIncomingStateMessage(msg,len); -#endif -} - -void Node::clusterStatus(ZT_ClusterStatus *cs) -{ - if (!cs) - return; -#ifdef ZT_ENABLE_CLUSTER - if (RR->cluster) - RR->cluster->status(*cs); - else -#endif - memset(cs,0,sizeof(ZT_ClusterStatus)); -} -*/ - /****************************************************************************/ /* Node methods used only within node/ */ /****************************************************************************/ @@ -918,7 +841,7 @@ enum ZT_ResultCode ZT_Node_processStateUpdate( ZT_Node *node, void *tptr, ZT_StateObjectType type, - uint64_t id, + const uint64_t id[2], const void *data, unsigned int len) { diff --git a/node/Node.hpp b/node/Node.hpp index f1209d00..17050d24 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -85,7 +85,7 @@ public: ZT_ResultCode processStateUpdate( void *tptr, ZT_StateObjectType type, - uint64_t id, + const uint64_t id[2], const void *data, unsigned int len); ZT_ResultCode processWirePacket( diff --git a/node/Path.hpp b/node/Path.hpp index 74b31d8d..a6f56d31 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -46,11 +46,6 @@ */ #define ZT_PATH_MAX_PREFERENCE_RANK ((ZT_INETADDRESS_MAX_SCOPE << 1) | 1) -/** - * Maximum distance for a path - */ -#define ZT_PATH_DISTANCE_MAX 0xffff - namespace ZeroTier { class RuntimeEnvironment; @@ -125,7 +120,6 @@ public: _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), - _distance(ZT_PATH_DISTANCE_MAX), _addr(), _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) @@ -143,7 +137,6 @@ public: _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), - _distance(ZT_PATH_DISTANCE_MAX), _addr(addr), _localAddress(localAddress), _ipScope(addr.ipScope()) @@ -311,11 +304,6 @@ public: */ inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } - /** - * @return Distance (higher is further) - */ - inline unsigned int distance() const { return _distance; } - /** * @param lo Last out send * @param li Last in send @@ -344,7 +332,6 @@ private: volatile signed int _incomingLinkQualitySlowLogCounter; volatile unsigned int _incomingLinkQualityPreviousPacketCounter; volatile unsigned int _outgoingPacketCounter; - volatile unsigned int _distance; InetAddress _addr; InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often diff --git a/node/Peer.cpp b/node/Peer.cpp index a7466296..18d05875 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -146,8 +146,8 @@ void Peer::received( path->updateLinkQuality((unsigned int)(packetId & 7)); if (hops == 0) { + // If this is a direct packet (no hops), update existing paths or learn new ones bool pathAlreadyKnown = false; - bool newPathLearned = false; { Mutex::Lock _l(_paths_m); @@ -188,7 +188,7 @@ void Peer::received( if (verb == Packet::VERB_OK) { potentialNewPeerPath->lr = now; potentialNewPeerPath->p = path; - newPathLearned = true; + _lastWroteState = 0; // force state write now } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); @@ -196,9 +196,6 @@ void Peer::received( } } } - - if (newPathLearned) - writeState(tPtr,now); } else if (this->trustEstablished(now)) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) { @@ -270,6 +267,9 @@ void Peer::received( } } } + + if ((now - _lastWroteState) > ZT_PEER_STATE_WRITE_PERIOD) + writeState(tPtr,now); } bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) @@ -435,7 +435,7 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) void Peer::writeState(void *tPtr,const uint64_t now) { try { - Buffer b; + Buffer b; b.append((uint8_t)1); // version b.append(now); @@ -455,7 +455,6 @@ void Peer::writeState(void *tPtr,const uint64_t now) b.append(_v4Path.p->lastOut()); b.append(_v4Path.p->lastIn()); b.append(_v4Path.p->lastTrustEstablishedPacketReceived()); - b.append((uint16_t)_v4Path.p->distance()); _v4Path.p->address().serialize(b); _v4Path.p->localAddress().serialize(b); } @@ -464,29 +463,29 @@ void Peer::writeState(void *tPtr,const uint64_t now) b.append(_v6Path.p->lastOut()); b.append(_v6Path.p->lastIn()); b.append(_v6Path.p->lastTrustEstablishedPacketReceived()); - b.append((uint16_t)_v6Path.p->distance()); _v6Path.p->address().serialize(b); _v6Path.p->localAddress().serialize(b); } } - b.append(_lastReceive); - b.append(_lastNontrivialReceive); - b.append(_lastTriedMemorizedPath); - b.append(_lastDirectPathPushSent); - b.append(_lastDirectPathPushReceive); - b.append(_lastCredentialRequestSent); - b.append(_lastWhoisRequestReceived); - b.append(_lastEchoRequestReceived); - b.append(_lastComRequestReceived); - b.append(_lastComRequestSent); - b.append(_lastCredentialsReceived); - b.append(_lastTrustEstablishedPacketReceived); - - b.append(_vProto); - b.append(_vMajor); - b.append(_vMinor); - b.append(_vRevision); + // Save space by sending these as time since now at 100ms resolution + b.append((uint16_t)(std::max(now - _lastReceive,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastNontrivialReceive,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastTriedMemorizedPath,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastDirectPathPushSent,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastDirectPathPushReceive,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastCredentialRequestSent,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastWhoisRequestReceived,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastEchoRequestReceived,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastComRequestReceived,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastComRequestSent,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastCredentialsReceived,(uint64_t)6553500) / 100)); + b.append((uint16_t)(std::max(now - _lastTrustEstablishedPacketReceived,(uint64_t)6553500) / 100)); + + b.append((uint8_t)_vProto); + b.append((uint8_t)_vMajor); + b.append((uint8_t)_vMinor); + b.append((uint16_t)_vRevision); b.append((uint16_t)0); // length of additional fields @@ -501,7 +500,7 @@ void Peer::writeState(void *tPtr,const uint64_t now) bool Peer::applyStateUpdate(const void *data,unsigned int len) { try { - Buffer b(data,len); + Buffer b(data,len); unsigned int ptr = 0; if (b[ptr++] != 1) @@ -510,6 +509,11 @@ bool Peer::applyStateUpdate(const void *data,unsigned int len) if (ts <= _lastReceivedStateTimestamp) return false; + Identity id; + ptr += id.deserialize(b,ptr); + if (id != _id) // sanity check + return false; + const unsigned int pathCount = (unsigned int)b[ptr++]; { Mutex::Lock _l(_paths_m); @@ -518,7 +522,6 @@ bool Peer::applyStateUpdate(const void *data,unsigned int len) const uint64_t lastOut = b.at(ptr); ptr += 8; const uint64_t lastIn = b.at(ptr); ptr += 8; const uint64_t lastTrustEstablishedPacketReceived = b.at(ptr); ptr += 8; - const unsigned int distance = b.at(ptr); ptr += 2; InetAddress addr,localAddr; ptr += addr.deserialize(b,ptr); ptr += localAddr.deserialize(b,ptr); @@ -529,8 +532,9 @@ bool Peer::applyStateUpdate(const void *data,unsigned int len) case AF_INET6: p = &_v6Path; break; } if (p) { - if ( ((p->p->address() != addr)||(p->p->localAddress() != localAddr)) && (p->p->distance() > distance) ) + if ( (!p->p) || ((p->p->address() != addr)||(p->p->localAddress() != localAddr)) ) { p->p = RR->topology->getPath(localAddr,addr); + } p->lr = lr; p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); } @@ -538,22 +542,22 @@ bool Peer::applyStateUpdate(const void *data,unsigned int len) } } - _lastReceive = std::max(_lastReceive,b.at(ptr)); ptr += 8; - _lastNontrivialReceive = std::max(_lastNontrivialReceive,b.at(ptr)); ptr += 8; - _lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,b.at(ptr)); ptr += 8; - _lastDirectPathPushSent = std::max(_lastDirectPathPushSent,b.at(ptr)); ptr += 8; - _lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,b.at(ptr)); ptr += 8; - _lastCredentialRequestSent = std::max(_lastCredentialRequestSent,b.at(ptr)); ptr += 8; - _lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,b.at(ptr)); ptr += 8; - _lastEchoRequestReceived = std::max(_lastEchoRequestReceived,b.at(ptr)); ptr += 8; - _lastComRequestReceived = std::max(_lastComRequestReceived,b.at(ptr)); ptr += 8; - _lastComRequestSent = std::max(_lastComRequestSent,b.at(ptr)); ptr += 8; - _lastCredentialsReceived = std::max(_lastCredentialsReceived,b.at(ptr)); ptr += 8; - _lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,b.at(ptr)); ptr += 8; - - _vProto = b.at(ptr); ptr += 2; - _vMajor = b.at(ptr); ptr += 2; - _vMinor = b.at(ptr); ptr += 2; + _lastReceive = std::max(_lastReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastNontrivialReceive = std::max(_lastNontrivialReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastDirectPathPushSent = std::max(_lastDirectPathPushSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastCredentialRequestSent = std::max(_lastCredentialRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastEchoRequestReceived = std::max(_lastEchoRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastComRequestReceived = std::max(_lastComRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastComRequestSent = std::max(_lastComRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastCredentialsReceived = std::max(_lastCredentialsReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + _lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; + + _vProto = (uint16_t)b[ptr++]; + _vMajor = (uint16_t)b[ptr++]; + _vMinor = (uint16_t)b[ptr++]; _vRevision = b.at(ptr); ptr += 2; _lastReceivedStateTimestamp = ts; @@ -563,4 +567,25 @@ bool Peer::applyStateUpdate(const void *data,unsigned int len) return false; } +SharedPtr Peer::createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len) +{ + try { + Identity id; + { + Buffer b(data,len); + unsigned int ptr = 0; + if (b[ptr++] != 1) + return SharedPtr(); + ptr += 8; // skip TS, don't care + id.deserialize(b,ptr); + } + if (id) { + const SharedPtr p(new Peer(renv,renv->identity,id)); + if (p->applyStateUpdate(data,len)) + return renv->topology->addPeer(tPtr,p); + } + } catch ( ... ) {} + return SharedPtr(); +} + } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index d6b7dad9..f0eb3ee8 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -51,6 +51,8 @@ #include "Mutex.hpp" #include "NonCopyable.hpp" +#define ZT_PEER_MAX_SERIALIZED_STATE_SIZE (sizeof(Peer) + 32 + (sizeof(Path) * 2)) + namespace ZeroTier { /** @@ -194,9 +196,10 @@ public: bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); /** - * Write current peer state to external storage / cluster network + * Write object state to external storage and/or cluster network * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param now Current time */ void writeState(void *tPtr,const uint64_t now); @@ -437,6 +440,17 @@ public: return false; } + /** + * Create a peer from a remote state update + * + * @param renv Runtime environment + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param data State update data + * @param len State update length + * @return Peer or NULL if data was invalid + */ + static SharedPtr createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len); + private: struct _PeerPath { diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index d8e1d699..ee0c8c24 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -30,8 +30,8 @@ #include #include "Constants.hpp" +#include "Utils.hpp" #include "Identity.hpp" -#include "Mutex.hpp" namespace ZeroTier { @@ -58,10 +58,13 @@ public: ,mc((Multicaster *)0) ,topology((Topology *)0) ,sa((SelfAwareness *)0) -#ifdef ZT_ENABLE_CLUSTER - ,cluster((Cluster *)0) -#endif { + Utils::getSecureRandom(&instanceId,sizeof(instanceId)); + } + + ~RuntimeEnvironment() + { + Utils::burn(reinterpret_cast(const_cast(secretIdentityStr.data())),(unsigned int)secretIdentityStr.length()); } // Node instance that owns this RuntimeEnvironment @@ -87,9 +90,11 @@ public: Multicaster *mc; Topology *topology; SelfAwareness *sa; -#ifdef ZT_ENABLE_CLUSTER - Cluster *cluster; -#endif + + /** + * A random integer identifying this run of ZeroTier + */ + uint32_t instanceId; }; } // namespace ZeroTier diff --git a/node/Switch.cpp b/node/Switch.cpp index 2be54b37..cbd73a83 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -108,13 +108,7 @@ void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAd const Address destination(fragment.destination()); if (destination != RR->identity.address()) { -#ifdef ZT_ENABLE_CLUSTER - const bool isClusterFrontplane = ((RR->cluster)&&(RR->cluster->isClusterPeerFrontplane(fromAddr))); -#else - const bool isClusterFrontplane = false; -#endif - - if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) && (!isClusterFrontplane) ) + if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) ) return; if (fragment.hops() < ZT_RELAY_MAX_HOPS) { @@ -124,13 +118,6 @@ void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAd // It wouldn't hurt anything, just redundant and unnecessary. SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); if ((!relayTo)||(!relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,false))) { -#ifdef ZT_ENABLE_CLUSTER - if ((RR->cluster)&&(!isClusterFrontplane)) { - RR->cluster->relayViaCluster(Address(),destination,fragment.data(),fragment.size(),false); - return; - } -#endif - // Don't know peer or no direct path -- so relay via someone upstream relayTo = RR->topology->getUpstreamPeer(); if (relayTo) @@ -197,13 +184,8 @@ void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAd //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); -#ifdef ZT_ENABLE_CLUSTER - if ( (source == RR->identity.address()) && ((!RR->cluster)||(!RR->cluster->isClusterPeerFrontplane(fromAddr))) ) - return; -#else if (source == RR->identity.address()) return; -#endif if (destination != RR->identity.address()) { if ( (!RR->topology->amRoot()) && (!path->trustEstablished(now)) && (source != RR->identity.address()) ) @@ -212,12 +194,7 @@ void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAd Packet packet(data,len); if (packet.hops() < ZT_RELAY_MAX_HOPS) { -#ifdef ZT_ENABLE_CLUSTER - if (source != RR->identity.address()) // don't increment hops for cluster frontplane relays - packet.incrementHops(); -#else packet.incrementHops(); -#endif SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); if ((relayTo)&&(relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,false))) { @@ -277,12 +254,6 @@ void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAd } } } else { -#ifdef ZT_ENABLE_CLUSTER - if ((RR->cluster)&&(source != RR->identity.address())) { - RR->cluster->relayViaCluster(source,destination,packet.data(),packet.size(),_shouldUnite(now,source,destination)); - return; - } -#endif relayTo = RR->topology->getUpstreamPeer(&source,1,true); if (relayTo) relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true); @@ -769,14 +740,6 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) const uint64_t now = RR->node->now(); const Address destination(packet.destination()); -#ifdef ZT_ENABLE_CLUSTER - uint64_t clusterMostRecentTs = 0; - int clusterMostRecentMemberId = -1; - uint8_t clusterPeerSecret[ZT_PEER_SECRET_KEY_LENGTH]; - if (RR->cluster) - clusterMostRecentMemberId = RR->cluster->checkSendViaCluster(destination,clusterMostRecentTs,clusterPeerSecret); -#endif - const SharedPtr peer(RR->topology->getPeer(tPtr,destination)); if (peer) { /* First get the best path, and if it's dead (and this is not a root) @@ -788,74 +751,37 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) viaPath = peer->getBestPath(now,false); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { -#ifdef ZT_ENABLE_CLUSTER - if ((clusterMostRecentMemberId < 0)||(viaPath->lastIn() > clusterMostRecentTs)) { -#endif - if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(tPtr,viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); - viaPath->sent(now); - } -#ifdef ZT_ENABLE_CLUSTER + if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { + peer->attemptToContactAt(tPtr,viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); + viaPath->sent(now); } -#endif viaPath.zero(); } -#ifdef ZT_ENABLE_CLUSTER - if (clusterMostRecentMemberId >= 0) { - if ((viaPath)&&(viaPath->lastIn() < clusterMostRecentTs)) - viaPath.zero(); - } else if (!viaPath) { -#else if (!viaPath) { -#endif peer->tryMemorizedPath(tPtr,now); // periodically attempt memorized or statically defined paths, if any are known const SharedPtr relay(RR->topology->getUpstreamPeer()); if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) { if (!(viaPath = peer->getBestPath(now,true))) return false; } -#ifdef ZT_ENABLE_CLUSTER } -#else - } -#endif } else { -#ifdef ZT_ENABLE_CLUSTER - if (clusterMostRecentMemberId < 0) { -#else - requestWhois(tPtr,destination); - return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly -#endif -#ifdef ZT_ENABLE_CLUSTER - } -#endif + requestWhois(tPtr,destination); + return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly } unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); packet.setFragmented(chunkSize < packet.size()); -#ifdef ZT_ENABLE_CLUSTER - const uint64_t trustedPathId = (viaPath) ? RR->topology->getOutboundPathTrust(viaPath->address()) : 0; - if (trustedPathId) { - packet.setTrusted(trustedPathId); - } else { - packet.armor((clusterMostRecentMemberId >= 0) ? clusterPeerSecret : peer->key(),encrypt,(viaPath) ? viaPath->nextOutgoingCounter() : 0); - } -#else const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address()); if (trustedPathId) { packet.setTrusted(trustedPathId); } else { packet.armor(peer->key(),encrypt,viaPath->nextOutgoingCounter()); } -#endif -#ifdef ZT_ENABLE_CLUSTER - if ( ((viaPath)&&(viaPath->send(RR,tPtr,packet.data(),chunkSize,now))) || ((clusterMostRecentMemberId >= 0)&&(RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,packet.data(),chunkSize))) ) { -#else if (viaPath->send(RR,tPtr,packet.data(),chunkSize,now)) { -#endif if (chunkSize < packet.size()) { // Too big for one packet, fragment the rest unsigned int fragStart = chunkSize; @@ -868,14 +794,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) for(unsigned int fno=1;fnosend(RR,tPtr,frag.data(),frag.size(),now); - else if (clusterMostRecentMemberId >= 0) - RR->cluster->sendViaCluster(clusterMostRecentMemberId,destination,frag.data(),frag.size()); -#else viaPath->send(RR,tPtr,frag.data(),frag.size(),now); -#endif fragStart += chunkSize; remaining -= chunkSize; } diff --git a/node/Topology.cpp b/node/Topology.cpp index be116b28..09a1a895 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -108,8 +108,6 @@ SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) np = hp; } - saveIdentity(tPtr,np->identity()); - return np; } @@ -128,18 +126,20 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) } try { - Identity id(_getIdentity(tPtr,zta)); - if (id) { - SharedPtr np(new Peer(RR,RR->identity,id)); - { - Mutex::Lock _l(_peers_m); - SharedPtr &ap = _peers[zta]; - if (!ap) - ap.swap(np); + char buf[ZT_PEER_MAX_SERIALIZED_STATE_SIZE]; + uint64_t idbuf[2]; idbuf[0] = zta.toInt(); idbuf[1] = 0; + int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_STATE,idbuf,buf,(unsigned int)sizeof(buf)); + if (len > 0) { + Mutex::Lock _l(_peers_m); + SharedPtr &ap = _peers[zta]; + if (ap) return ap; - } + ap = Peer::createFromStateUpdate(RR,tPtr,buf,len); + if (!ap) + _peers.erase(zta); + return ap; } - } catch ( ... ) {} // invalid identity on disk? + } catch ( ... ) {} // ignore invalid identities or other strage failures return SharedPtr(); } @@ -154,17 +154,7 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta) if (ap) return (*ap)->identity(); } - return _getIdentity(tPtr,zta); -} - -void Topology::saveIdentity(void *tPtr,const Identity &id) -{ - if (id) { - const std::string tmp(id.toString(false)); - uint64_t idtmp[2]; - idtmp[0] = id.address().toInt(); idtmp[1] = 0; - RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp.data(),(unsigned int)tmp.length()); - } + return Identity(); } SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid) @@ -423,21 +413,6 @@ void Topology::doPeriodicTasks(void *tPtr,uint64_t now) } } -Identity Topology::_getIdentity(void *tPtr,const Address &zta) -{ - char tmp[512]; - uint64_t idtmp[2]; - idtmp[0] = zta.toInt(); idtmp[1] = 0; - int n = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_IDENTITY,idtmp,tmp,sizeof(tmp) - 1); - if (n > 0) { - tmp[n] = (char)0; - try { - return Identity(tmp); - } catch ( ... ) {} // ignore invalid IDs - } - return Identity(); -} - void Topology::_memoizeUpstreams(void *tPtr) { // assumes _upstreams_m and _peers_m are locked @@ -450,10 +425,8 @@ void Topology::_memoizeUpstreams(void *tPtr) } else if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),i->identity.address()) == _upstreamAddresses.end()) { _upstreamAddresses.push_back(i->identity.address()); SharedPtr &hp = _peers[i->identity.address()]; - if (!hp) { + if (!hp) hp = new Peer(RR,RR->identity,i->identity); - saveIdentity(tPtr,i->identity); - } } } @@ -464,10 +437,8 @@ void Topology::_memoizeUpstreams(void *tPtr) } else if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),i->identity.address()) == _upstreamAddresses.end()) { _upstreamAddresses.push_back(i->identity.address()); SharedPtr &hp = _peers[i->identity.address()]; - if (!hp) { + if (!hp) hp = new Peer(RR,RR->identity,i->identity); - saveIdentity(tPtr,i->identity); - } } } } diff --git a/node/Topology.hpp b/node/Topology.hpp index 9bc7c0d8..32e38dd3 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -81,6 +81,13 @@ public: */ SharedPtr getPeer(void *tPtr,const Address &zta); + /** + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param zta ZeroTier address of peer + * @return Identity or NULL identity if not found + */ + Identity getIdentity(void *tPtr,const Address &zta); + /** * Get a peer only if it is presently in memory (no disk cache) * @@ -116,26 +123,6 @@ public: return p; } - /** - * Get the identity of a peer - * - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param zta ZeroTier address of peer - * @return Identity or NULL Identity if not found - */ - Identity getIdentity(void *tPtr,const Address &zta); - - /** - * Cache an identity - * - * This is done automatically on addPeer(), and so is only useful for - * cluster identity replication. - * - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param id Identity to cache - */ - void saveIdentity(void *tPtr,const Identity &id); - /** * Get the current best upstream peer * diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index a0b47367..b1fe5921 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -180,7 +180,7 @@ public: const unsigned long pid = (unsigned long)getpid(); // Get all device names - Utils::snprintf(fn,sizeof(fn),"/proc/%lu/net/dev",pid); + Utils::ztsnprintf(fn,sizeof(fn),"/proc/%lu/net/dev",pid); FILE *procf = fopen(fn,"r"); if (procf) { while (fgets(tmp,sizeof(tmp),procf)) { @@ -196,7 +196,7 @@ public: } // Get IPv6 addresses (and any device names we don't already know) - Utils::snprintf(fn,sizeof(fn),"/proc/%lu/net/if_inet6",pid); + Utils::ztsnprintf(fn,sizeof(fn),"/proc/%lu/net/if_inet6",pid); procf = fopen(fn,"r"); if (procf) { while (fgets(tmp,sizeof(tmp),procf)) { diff --git a/service/OneService.cpp b/service/OneService.cpp index f949f348..b5b11111 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -154,9 +154,6 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } // How often to check for local interface addresses #define ZT_LOCAL_INTERFACE_CHECK_INTERVAL 60000 -// Clean files from iddb.d that are older than this (60 days) -#define ZT_IDDB_CLEANUP_AGE 5184000000ULL - // Maximum write buffer size for outgoing TCP connections (sanity limit) #define ZT_TCP_MAX_WRITEQ_SIZE 33554432 @@ -414,7 +411,6 @@ public: const std::string _homePath; std::string _authToken; std::string _controllerDbPath; - const std::string _iddbPath; const std::string _networksPath; const std::string _moonsPath; @@ -513,7 +509,6 @@ public: OneServiceImpl(const char *hp,unsigned int port) : _homePath((hp) ? hp : ".") ,_controllerDbPath(_homePath + ZT_PATH_SEPARATOR_S "controller.d") - ,_iddbPath(_homePath + ZT_PATH_SEPARATOR_S "iddb.d") ,_networksPath(_homePath + ZT_PATH_SEPARATOR_S "networks.d") ,_moonsPath(_homePath + ZT_PATH_SEPARATOR_S "moons.d") ,_controller((EmbeddedNetworkController *)0) @@ -732,6 +727,9 @@ public: } #endif + // Delete legacy iddb.d if present (cleanup) + OSUtils::rmDashRf((_homePath + ZT_PATH_SEPARATOR_S "iddb.d").c_str()); + // Network controller is now enabled by default for desktop and server _controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str()); _node->setNetconfMaster((void *)_controller); @@ -781,7 +779,6 @@ public: uint64_t lastBindRefresh = 0; uint64_t lastUpdateCheck = clockShouldBe; uint64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle - uint64_t lastCleanedIddb = 0; uint64_t lastTcpCheck = 0; for(;;) { _run_m.lock(); @@ -797,12 +794,6 @@ public: const uint64_t now = OSUtils::now(); - // Clean iddb.d on start and every 24 hours - if ((now - lastCleanedIddb) > 86400000) { - lastCleanedIddb = now; - OSUtils::cleanDirectory(_iddbPath.c_str(),now - ZT_IDDB_CLEANUP_AGE); - } - // Attempt to detect sleep/wake events by detecting delay overruns bool restarted = false; if ((now > clockShouldBe)&&((now - clockShouldBe) > 10000)) { @@ -1027,7 +1018,7 @@ public: return NULL; } - virtual Node * getNode() + virtual Node *getNode() { return _node; } @@ -1903,27 +1894,16 @@ public: char *const outdata = const_cast(tc->writeq.data()) + startpos; encryptClusterMessage(outdata,mlen); - } - - void replicateStateObjectToCluster(const ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len,const uint64_t everyoneBut) - { - std::vector sentTo; - if (everyoneBut) - sentTo.push_back(everyoneBut); - Mutex::Lock _l(_tcpConnections_m); - for(std::vector::const_iterator ci(_tcpConnections.begin());ci!=_tcpConnections.end();++ci) { - TcpConnection *const c = *ci; - if ((c->type == TcpConnection::TCP_CLUSTER_BACKPLANE)&&(c->clusterMemberId != 0)&&(std::find(sentTo.begin(),sentTo.end(),c->clusterMemberId) == sentTo.end())) { - sentTo.push_back(c->clusterMemberId); - replicateStateObject(type,id,data,len,c); - } - } + tc->writeq.append(outdata,mlen); } void writeStateObject(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { - char p[4096]; + char buf[65535]; + char p[1024]; + FILE *f; bool secure = false; + switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); @@ -1932,13 +1912,14 @@ public: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); secure = true; break; - case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id[0]); - break; + //case ZT_STATE_OBJECT_PEER_STATE: + // break; case ZT_STATE_OBJECT_NETWORK_CONFIG: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); secure = true; break; + //case ZT_STATE_OBJECT_NETWORK_MEMBERSHIP: + // break; case ZT_STATE_OBJECT_PLANET: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; @@ -1949,17 +1930,30 @@ public: p[0] = (char)0; break; } + if (p[0]) { if (len >= 0) { - FILE *f = fopen(p,"w"); + // Check to see if we've already written this first. This reduces + // redundant writes and I/O overhead on most platforms and has + // little effect on others. + f = fopen(p,"r"); + bool redundant = false; if (f) { - if (fwrite(data,len,1,f) != 1) - fprintf(stderr,"WARNING: unable to write to file: %s (I/O error)" ZT_EOL_S,p); + long l = (long)fread(buf,1,sizeof(buf),f); fclose(f); - if (secure) - OSUtils::lockDownFile(p,false); - } else { - fprintf(stderr,"WARNING: unable to write to file: %s (unable to open)" ZT_EOL_S,p); + redundant = ((l == (long)len)&&(memcmp(data,buf,l) == 0)); + } + if (!redundant) { + f = fopen(p,"w"); + if (f) { + if (fwrite(data,len,1,f) != 1) + fprintf(stderr,"WARNING: unable to write to file: %s (I/O error)" ZT_EOL_S,p); + fclose(f); + if (secure) + OSUtils::lockDownFile(p,false); + } else { + fprintf(stderr,"WARNING: unable to write to file: %s (unable to open)" ZT_EOL_S,p); + } } } else { OSUtils::rm(p); @@ -2314,7 +2308,7 @@ public: break; case CLUSTER_MESSAGE_STATE_OBJECT: - if (mlen >= 42) { // type + object ID + [data] + if (mlen > 42) { // type + object ID + [data] uint64_t objId[2]; objId[0] = ( ((uint64_t)data[26] << 56) | @@ -2336,10 +2330,8 @@ public: ((uint64_t)data[40] << 8) | (uint64_t)data[41] ); - if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId[0],data + 42,(unsigned int)(mlen - 42)) == ZT_RESULT_OK) { + if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42)) == ZT_RESULT_OK) writeStateObject((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42)); - replicateStateObjectToCluster((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42),tc->clusterMemberId); - } } break; @@ -2558,7 +2550,18 @@ public: inline void nodeStatePutFunction(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { writeStateObject(type,id,data,len); - replicateStateObjectToCluster(type,id,data,len,0); + + std::vector sentTo; + { + Mutex::Lock _l(_tcpConnections_m); + for(std::vector::const_iterator ci(_tcpConnections.begin());ci!=_tcpConnections.end();++ci) { + TcpConnection *const c = *ci; + if ((c->type == TcpConnection::TCP_CLUSTER_BACKPLANE)&&(c->clusterMemberId != 0)&&(std::find(sentTo.begin(),sentTo.end(),c->clusterMemberId) == sentTo.end())) { + sentTo.push_back(c->clusterMemberId); + replicateStateObject(type,id,data,len,c); + } + } + } } inline int nodeStateGetFunction(enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen) @@ -2571,9 +2574,6 @@ public: case ZT_STATE_OBJECT_IDENTITY_SECRET: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); break; - case ZT_STATE_OBJECT_PEER_IDENTITY: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "iddb.d/%.10llx",_homePath.c_str(),(unsigned long long)id); - break; case ZT_STATE_OBJECT_NETWORK_CONFIG: Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); break; -- cgit v1.2.3 From f18158a52d28c14352018a68d328f41fcdb7966f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 11:45:22 -0700 Subject: . --- include/ZeroTierOne.h | 135 ++-------- node/IncomingPacket.cpp | 14 +- node/Node.cpp | 130 +--------- node/Node.hpp | 14 +- node/Path.cpp | 2 +- node/Path.hpp | 51 ++-- node/Peer.cpp | 59 ++--- node/Peer.hpp | 12 +- node/RuntimeEnvironment.hpp | 10 +- node/SelfAwareness.cpp | 4 +- node/SelfAwareness.hpp | 10 +- node/Switch.cpp | 8 +- node/Switch.hpp | 4 +- node/Topology.cpp | 4 +- node/Topology.hpp | 4 +- osdep/Binder.hpp | 109 ++------ service/OneService.cpp | 609 +++++--------------------------------------- 17 files changed, 198 insertions(+), 981 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 133ae340..180e5cd2 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -229,11 +229,6 @@ extern "C" { */ #define ZT_RULE_PACKET_CHARACTERISTICS_TCP_FIN 0x0000000000000001ULL -/** - * A null/empty sockaddr (all zero) to signify an unspecified socket address - */ -extern const struct sockaddr_storage ZT_SOCKADDR_NULL; - /****************************************************************************/ /* Structures and other types */ /****************************************************************************/ @@ -1067,21 +1062,6 @@ typedef struct /** * ZeroTier core state objects - * - * All of these objects can be persisted if desired. To preserve the - * identity of a node and its address, the identity (public and secret) - * must be saved at a minimum. - * - * State objects actually have two IDs (uint64_t[2]). If only one is - * listed the second ([1]) should be zero and is ignored in storage - * and replication. - * - * All state objects should be replicated in cluster mode. The reference - * clustering implementation uses a rumor mill algorithm in which state - * updates that are accepted with RESULT_OK (but not RESULT_OK_IGNORED) - * are flooded to all connected cluster peers. This results in updates - * being flooded across the cluster until all cluster members have the - * latest. */ enum ZT_StateObjectType { @@ -1108,36 +1088,6 @@ enum ZT_StateObjectType */ ZT_STATE_OBJECT_IDENTITY_SECRET = 2, - /** - * A peer to which this node is communicating - * - * Object ID: peer address - * Canonical path: /peers.d/
(10-digit hex address) - * Persistence: optional, can be purged at any time - */ - ZT_STATE_OBJECT_PEER_STATE = 3, - - /** - * Network configuration - * - * Object ID: peer address - * Canonical path: /networks.d/.conf (16-digit hex ID) - * Persistence: required if network memberships should persist - */ - ZT_STATE_OBJECT_NETWORK_CONFIG = 4, - - /** - * Network membership (network X peer intersection) - * - * If these are persisted they must be restored after peer states and - * network configs. Otherwise they are ignored. - * - * Object ID: [0] network ID, [1] peer address - * Canonical path: /networks.d//members.d/
- * Persistence: optional (not usually needed) - */ - ZT_STATE_OBJECT_NETWORK_MEMBERSHIP = 5, - /** * The planet (there is only one per... well... planet!) * @@ -1145,7 +1095,7 @@ enum ZT_StateObjectType * Canonical path: /planet * Persistence: recommended */ - ZT_STATE_OBJECT_PLANET = 6, + ZT_STATE_OBJECT_PLANET = 3, /** * A moon (federated root set) @@ -1154,12 +1104,25 @@ enum ZT_StateObjectType * Canonical path: /moons.d/.moon (16-digit hex ID) * Persistence: required if moon memberships should persist */ - ZT_STATE_OBJECT_MOON = 7, + ZT_STATE_OBJECT_MOON = 4, /** - * IDs above this value will not be used by the core (and could be used as implementation-specific IDs) + * Peer and related state + * + * Object ID: peer address + * Canonical path: /peers.d/ (10-digit address + * Persistence: optional, can be cleared at any time + */ + ZT_STATE_OBJECT_PEER = 5, + + /** + * Network configuration + * + * Object ID: peer address + * Canonical path: /networks.d/.conf (16-digit hex ID) + * Persistence: required if network memberships should persist */ - ZT_STATE_OBJECT__MAX_ID = 255 + ZT_STATE_OBJECT_NETWORK_CONFIG = 6 }; /** @@ -1277,17 +1240,15 @@ typedef int (*ZT_StateGetFunction)( * Parameters: * (1) Node * (2) User pointer - * (3) Local interface address + * (3) Local socket or -1 for "all" or "any" * (4) Remote address * (5) Packet data * (6) Packet length * (7) Desired IP TTL or 0 to use default * - * If there is only one local interface it is safe to ignore the local - * interface address. Otherwise if running with multiple interfaces, the - * correct local interface should be chosen by address unless NULL. If - * the ss_family field is zero (NULL address), a random or preferred - * default interface should be used. + * If there is only one local socket, the local socket can be ignored. + * If the local socket is -1, the packet should be sent out from all + * bound local sockets or a random bound local socket. * * If TTL is nonzero, packets should have their IP TTL value set to this * value if possible. If this is not possible it is acceptable to ignore @@ -1301,7 +1262,7 @@ typedef int (*ZT_WirePacketSendFunction)( ZT_Node *, /* Node */ void *, /* User ptr */ void *, /* Thread ptr */ - const struct sockaddr_storage *, /* Local address */ + int64_t, /* Local socket */ const struct sockaddr_storage *, /* Remote address */ const void *, /* Packet data */ unsigned int, /* Packet length */ @@ -1314,7 +1275,7 @@ typedef int (*ZT_WirePacketSendFunction)( * (1) Node * (2) User pointer * (3) ZeroTier address or 0 for none/any - * (4) Local interface address + * (4) Local socket or -1 if unknown * (5) Remote address * * This function must return nonzero (true) if the path should be used. @@ -1333,7 +1294,7 @@ typedef int (*ZT_PathCheckFunction)( void *, /* User ptr */ void *, /* Thread ptr */ uint64_t, /* ZeroTier address */ - const struct sockaddr_storage *, /* Local address */ + int64_t, /* Local socket or -1 if unknown */ const struct sockaddr_storage *); /* Remote address */ /** @@ -1441,57 +1402,13 @@ enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct */ void ZT_Node_delete(ZT_Node *node); -/** - * Notify node of an update to a state object - * - * This can be called after node startup to restore cached state objects such - * as network configurations for joined networks, planet, moons, etc. See - * the documentation of ZT_StateObjectType for more information. It's okay - * to call this for everything in the object store, but note that the node - * will automatically query for some core objects like identities so supplying - * these via this function is not necessary. - * - * Unless clustering is being implemented this function doesn't need to be - * used after startup. It could be called in response to filesystem changes - * to allow some degree of live configurability by filesystem observation - * but this kind of thing is entirely optional. - * - * The return value of this function indicates whether the update was accepted - * as new. A return value of ZT_RESULT_OK indicates that the node gleaned new - * information from this update and that therefore (in cluster rumor mill mode) - * this update should be distributed to other members of a cluster. A return - * value of ZT_RESULT_OK_IGNORED indicates that the object did not provide any - * new information and therefore should not be propagated in a cluster. - * - * If clustering isn't being implemented the return value of this function can - * generally be ignored. - * - * ZT_RESULT_ERROR_BAD_PARAMETER can be returned if the parameter was invalid - * or not applicable. Object stores may delete the object in this case. - * - * @param node Node instance - * @param tptr Thread pointer to pass to functions/callbacks resulting from this call - * @param type State object type - * @param id State object ID (if object type has only one ID, second should be zero) - * @param data State object data - * @param len Length of state object data in bytes - * @return ZT_RESULT_OK if object was accepted or ZT_RESULT_OK_IGNORED if non-informative, error if object was invalid - */ -enum ZT_ResultCode ZT_Node_processStateUpdate( - ZT_Node *node, - void *tptr, - ZT_StateObjectType type, - const uint64_t id[2], - const void *data, - unsigned int len); - /** * Process a packet received from the physical wire * * @param node Node instance * @param tptr Thread pointer to pass to functions/callbacks resulting from this call * @param now Current clock in milliseconds - * @param localAddress Local address, or point to ZT_SOCKADDR_NULL if unspecified + * @param localSocket Local socket (you can use 0 if only one local socket is bound and ignore this) * @param remoteAddress Origin of packet * @param packetData Packet data * @param packetLength Packet length @@ -1502,7 +1419,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, uint64_t now, - const struct sockaddr_storage *localAddress, + int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 0548387b..f0be96f9 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -309,7 +309,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool if (ptr < size()) { ptr += externalSurfaceAddress.deserialize(*this,ptr); if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(tPtr,id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + RR->sa->iam(tPtr,id.address(),_path->localSocket(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); } // Get primary planet world ID and world timestamp if present @@ -495,7 +495,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(tPtr,peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); + RR->sa->iam(tPtr,peer->address(),_path->localSocket(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); } break; case Packet::VERB_WHOIS: @@ -613,9 +613,9 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localAddress(),atAddr)) { - RR->node->putPacket(tPtr,_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(tPtr,_path->localAddress(),atAddr,RR->node->now(),false,0); + if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localSocket(),atAddr)) { + RR->node->putPacket(tPtr,_path->localSocket(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->attemptToContactAt(tPtr,_path->localSocket(),atAddr,RR->node->now(),false,0); TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } else { TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); @@ -1197,7 +1197,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known - (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { //if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) // peer->setClusterPreferred(a); @@ -1214,7 +1214,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known - (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { //if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) // peer->setClusterPreferred(a); diff --git a/node/Node.cpp b/node/Node.cpp index 4ffe496c..4b598f61 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -47,8 +47,6 @@ #include "SelfAwareness.hpp" #include "Network.hpp" -const struct sockaddr_storage ZT_SOCKADDR_NULL = {0}; - namespace ZeroTier { /****************************************************************************/ @@ -137,114 +135,17 @@ Node::~Node() delete RR->sw; } -ZT_ResultCode Node::processStateUpdate( - void *tptr, - ZT_StateObjectType type, - const uint64_t id[2], - const void *data, - unsigned int len) -{ - ZT_ResultCode r = ZT_RESULT_OK_IGNORED; - switch(type) { - - case ZT_STATE_OBJECT_PEER_STATE: - if (len) { - const SharedPtr p(RR->topology->getPeer(tptr,Address(id[0]))); - if (p) { - r = (p->applyStateUpdate(data,len)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED; - } else { - r = (Peer::createFromStateUpdate(RR,tptr,data,len)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED; - } - } - break; - - case ZT_STATE_OBJECT_NETWORK_CONFIG: - if (len <= (ZT_NETWORKCONFIG_DICT_CAPACITY - 1)) { - if (len < 2) { - Mutex::Lock _l(_networks_m); - SharedPtr &nw = _networks[id[0]]; - if (!nw) { - nw = SharedPtr(new Network(RR,tptr,id[0],(void *)0,(const NetworkConfig *)0)); - r = ZT_RESULT_OK; - } - } else { - Dictionary *dict = new Dictionary(reinterpret_cast(data),len); - try { - NetworkConfig *nconf = new NetworkConfig(); - try { - if (nconf->fromDictionary(*dict)) { - Mutex::Lock _l(_networks_m); - SharedPtr &nw = _networks[id[0]]; - if (nw) { - switch (nw->setConfiguration(tptr,*nconf,false)) { - default: - r = ZT_RESULT_ERROR_BAD_PARAMETER; - break; - case 1: - r = ZT_RESULT_OK_IGNORED; - break; - case 2: - r = ZT_RESULT_OK; - break; - } - } else { - nw = SharedPtr(new Network(RR,tptr,id[0],(void *)0,nconf)); - } - } else { - r = ZT_RESULT_ERROR_BAD_PARAMETER; - } - } catch ( ... ) { - r = ZT_RESULT_ERROR_BAD_PARAMETER; - } - delete nconf; - } catch ( ... ) { - r = ZT_RESULT_ERROR_BAD_PARAMETER; - } - delete dict; - } - } else { - r = ZT_RESULT_ERROR_BAD_PARAMETER; - } - break; - - case ZT_STATE_OBJECT_NETWORK_MEMBERSHIP: - if (len) { - } - break; - - case ZT_STATE_OBJECT_PLANET: - case ZT_STATE_OBJECT_MOON: - if ((len)&&(len <= ZT_WORLD_MAX_SERIALIZED_LENGTH)) { - World w; - try { - w.deserialize(Buffer(data,len)); - if (( (w.type() == World::TYPE_MOON)&&(type == ZT_STATE_OBJECT_MOON) )||( (w.type() == World::TYPE_PLANET)&&(type == ZT_STATE_OBJECT_PLANET) )) { - r = (RR->topology->addWorld(tptr,w,false)) ? ZT_RESULT_OK : ZT_RESULT_OK_IGNORED; - } - } catch ( ... ) { - r = ZT_RESULT_ERROR_BAD_PARAMETER; - } - } else { - r = ZT_RESULT_ERROR_BAD_PARAMETER; - } - break; - - default: break; - } - return r; -} - ZT_ResultCode Node::processWirePacket( void *tptr, uint64_t now, - const struct sockaddr_storage *localAddress, + int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, volatile uint64_t *nextBackgroundTaskDeadline) { _now = now; - RR->sw->onRemotePacket(tptr,*(reinterpret_cast(localAddress)),*(reinterpret_cast(remoteAddress)),packetData,packetLength); + RR->sw->onRemotePacket(tptr,localSocket,*(reinterpret_cast(remoteAddress)),packetData,packetLength); return ZT_RESULT_OK; } @@ -317,7 +218,7 @@ public: if ((!contacted)&&(_bestCurrentUpstream)) { const SharedPtr up(_bestCurrentUpstream->getBestPath(_now,true)); if (up) - p->sendHELLO(_tPtr,up->localAddress(),up->address(),_now,up->nextOutgoingCounter()); + p->sendHELLO(_tPtr,up->localSocket(),up->address(),_now,up->nextOutgoingCounter()); } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); @@ -617,7 +518,7 @@ void Node::setNetconfMaster(void *networkControllerInstance) /* Node methods used only within node/ */ /****************************************************************************/ -bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress) +bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const int64_t localSocket,const InetAddress &remoteAddress) { if (!Path::isAddressValidForPath(remoteAddress)) return false; @@ -640,7 +541,7 @@ bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,cons } } - return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),reinterpret_cast(&localAddress),reinterpret_cast(&remoteAddress)) != 0) : true); + return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),localSocket,reinterpret_cast(&remoteAddress)) != 0) : true); } #ifdef ZT_TRACE @@ -837,35 +738,18 @@ void ZT_Node_delete(ZT_Node *node) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_processStateUpdate( - ZT_Node *node, - void *tptr, - ZT_StateObjectType type, - const uint64_t id[2], - const void *data, - unsigned int len) -{ - try { - return reinterpret_cast(node)->processStateUpdate(tptr,type,id,data,len); - } catch (std::bad_alloc &exc) { - return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, uint64_t now, - const struct sockaddr_storage *localAddress, + int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, volatile uint64_t *nextBackgroundTaskDeadline) { try { - return reinterpret_cast(node)->processWirePacket(tptr,now,localAddress,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); + return reinterpret_cast(node)->processWirePacket(tptr,now,localSocket,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); } catch (std::bad_alloc &exc) { return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY; } catch ( ... ) { diff --git a/node/Node.hpp b/node/Node.hpp index 17050d24..55491b06 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -82,16 +82,10 @@ public: // Public API Functions ---------------------------------------------------- - ZT_ResultCode processStateUpdate( - void *tptr, - ZT_StateObjectType type, - const uint64_t id[2], - const void *data, - unsigned int len); ZT_ResultCode processWirePacket( void *tptr, uint64_t now, - const struct sockaddr_storage *localAddress, + int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, @@ -129,13 +123,13 @@ public: inline uint64_t now() const throw() { return _now; } - inline bool putPacket(void *tPtr,const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) + inline bool putPacket(void *tPtr,const int64_t localSocket,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { return (_cb.wirePacketSendFunction( reinterpret_cast(this), _uPtr, tPtr, - reinterpret_cast(&localAddress), + localSocket, reinterpret_cast(&addr), data, len, @@ -205,7 +199,7 @@ public: void postTrace(const char *module,unsigned int line,const char *fmt,...); #endif - bool shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress); + bool shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const int64_t localSocket,const InetAddress &remoteAddress); inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); diff --git a/node/Path.cpp b/node/Path.cpp index a5fe1aa7..9dc9aba5 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -32,7 +32,7 @@ namespace ZeroTier { bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now) { - if (RR->node->putPacket(tPtr,_localAddress,address(),data,len)) { + if (RR->node->putPacket(tPtr,_localSocket,_addr,data,len)) { _lastOut = now; return true; } diff --git a/node/Path.hpp b/node/Path.hpp index a6f56d31..854b28e2 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -66,49 +66,28 @@ public: public: HashKey() {} - HashKey(const InetAddress &l,const InetAddress &r) + HashKey(const int64_t l,const InetAddress &r) { - // This is an ad-hoc bit packing algorithm to yield unique keys for - // remote addresses and their local-side counterparts if defined. - // Portability across runtimes is not needed. if (r.ss_family == AF_INET) { _k[0] = (uint64_t)reinterpret_cast(&r)->sin_addr.s_addr; _k[1] = (uint64_t)reinterpret_cast(&r)->sin_port; - if (l.ss_family == AF_INET) { - _k[2] = (uint64_t)reinterpret_cast(&l)->sin_addr.s_addr; - _k[3] = (uint64_t)reinterpret_cast(&r)->sin_port; - } else { - _k[2] = 0; - _k[3] = 0; - } + _k[2] = (uint64_t)l; } else if (r.ss_family == AF_INET6) { - const uint8_t *a = reinterpret_cast(reinterpret_cast(&r)->sin6_addr.s6_addr); - uint8_t *b = reinterpret_cast(_k); - for(unsigned int i=0;i<16;++i) b[i] = a[i]; - _k[2] = ~((uint64_t)reinterpret_cast(&r)->sin6_port); - if (l.ss_family == AF_INET6) { - _k[2] ^= ((uint64_t)reinterpret_cast(&r)->sin6_port) << 32; - a = reinterpret_cast(reinterpret_cast(&l)->sin6_addr.s6_addr); - b += 24; - for(unsigned int i=0;i<8;++i) b[i] = a[i]; - a += 8; - for(unsigned int i=0;i<8;++i) b[i] ^= a[i]; - } + memcpy(_k,reinterpret_cast(&r)->sin6_addr.s6_addr,16); + _k[2] = ((uint64_t)reinterpret_cast(&r)->sin6_port << 32) ^ (uint64_t)l; } else { - _k[0] = 0; - _k[1] = 0; - _k[2] = 0; - _k[3] = 0; + memcpy(_k,&r,std::min(sizeof(_k),sizeof(InetAddress))); + _k[2] += (uint64_t)l; } } - inline unsigned long hashCode() const { return (unsigned long)(_k[0] + _k[1] + _k[2] + _k[3]); } + inline unsigned long hashCode() const { return (unsigned long)(_k[0] + _k[1] + _k[2]); } - inline bool operator==(const HashKey &k) const { return ( (_k[0] == k._k[0]) && (_k[1] == k._k[1]) && (_k[2] == k._k[2]) && (_k[3] == k._k[3]) ); } + inline bool operator==(const HashKey &k) const { return ( (_k[0] == k._k[0]) && (_k[1] == k._k[1]) && (_k[2] == k._k[2]) ); } inline bool operator!=(const HashKey &k) const { return (!(*this == k)); } private: - uint64_t _k[4]; + uint64_t _k[3]; }; Path() : @@ -116,29 +95,29 @@ public: _lastIn(0), _lastTrustEstablishedPacketReceived(0), _incomingLinkQualityFastLog(0xffffffffffffffffULL), + _localSocket(-1), _incomingLinkQualitySlowLogPtr(0), _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), _addr(), - _localAddress(), _ipScope(InetAddress::IP_SCOPE_NONE) { for(int i=0;i<(int)sizeof(_incomingLinkQualitySlowLog);++i) _incomingLinkQualitySlowLog[i] = ZT_PATH_LINK_QUALITY_MAX; } - Path(const InetAddress &localAddress,const InetAddress &addr) : + Path(const int64_t localSocket,const InetAddress &addr) : _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), _incomingLinkQualityFastLog(0xffffffffffffffffULL), + _localSocket(localSocket), _incomingLinkQualitySlowLogPtr(0), _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), _addr(addr), - _localAddress(localAddress), _ipScope(addr.ipScope()) { for(int i=0;i<(int)sizeof(_incomingLinkQualitySlowLog);++i) @@ -210,9 +189,9 @@ public: inline void sent(const uint64_t t) { _lastOut = t; } /** - * @return Address of local side of this path or NULL if unspecified + * @return Local socket as specified by external code */ - inline const InetAddress &localAddress() const { return _localAddress; } + inline const int64_t localSocket() const { return _localSocket; } /** * @return Physical address @@ -328,12 +307,12 @@ private: volatile uint64_t _lastIn; volatile uint64_t _lastTrustEstablishedPacketReceived; volatile uint64_t _incomingLinkQualityFastLog; + int64_t _localSocket; volatile unsigned long _incomingLinkQualitySlowLogPtr; volatile signed int _incomingLinkQualitySlowLogCounter; volatile unsigned int _incomingLinkQualityPreviousPacketCounter; volatile unsigned int _outgoingPacketCounter; InetAddress _addr; - InetAddress _localAddress; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often volatile uint8_t _incomingLinkQualitySlowLog[32]; AtomicCounter __refCount; diff --git a/node/Peer.cpp b/node/Peer.cpp index 18d05875..875d651e 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -154,25 +154,21 @@ void Peer::received( if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) { const struct sockaddr_in *const r = reinterpret_cast(&(path->address())); const struct sockaddr_in *const l = reinterpret_cast(&(_v4Path.p->address())); - const struct sockaddr_in *const rl = reinterpret_cast(&(path->localAddress())); - const struct sockaddr_in *const ll = reinterpret_cast(&(_v4Path.p->localAddress())); - if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(rl->sin_addr.s_addr == ll->sin_addr.s_addr)&&(rl->sin_port == ll->sin_port)) { + if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(path->localSocket() == _v4Path.p->localSocket())) { _v4Path.lr = now; pathAlreadyKnown = true; } } else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) { const struct sockaddr_in6 *const r = reinterpret_cast(&(path->address())); const struct sockaddr_in6 *const l = reinterpret_cast(&(_v6Path.p->address())); - const struct sockaddr_in6 *const rl = reinterpret_cast(&(path->localAddress())); - const struct sockaddr_in6 *const ll = reinterpret_cast(&(_v6Path.p->localAddress())); - if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(!memcmp(rl->sin6_addr.s6_addr,ll->sin6_addr.s6_addr,16))&&(rl->sin6_port == ll->sin6_port)) { + if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(path->localSocket() == _v6Path.p->localSocket())) { _v6Path.lr = now; pathAlreadyKnown = true; } } } - if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { + if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address())) ) { Mutex::Lock _l(_paths_m); _PeerPath *potentialNewPeerPath = (_PeerPath *)0; if (path->address().ss_family == AF_INET) { @@ -191,7 +187,7 @@ void Peer::received( _lastWroteState = 0; // force state write now } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); + attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); path->sent(now); } } @@ -318,7 +314,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) return SharedPtr(); } -void Peer::sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) +void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,unsigned int counter) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -360,21 +356,21 @@ void Peer::sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress & if (atAddress) { outp.armor(_key,false,counter); // false == don't encrypt full payload, but add MAC - RR->node->putPacket(tPtr,localAddr,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); } else { RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC } } -void Peer::attemptToContactAt(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) +void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) { if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); RR->node->expectReplyTo(outp.packetId()); outp.armor(_key,true,counter); - RR->node->putPacket(tPtr,localAddr,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); } else { - sendHELLO(tPtr,localAddr,atAddress,now,counter); + sendHELLO(tPtr,localSocket,atAddress,now,counter); } } @@ -402,13 +398,13 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) if (v6lr > v4lr) { if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + attemptToContactAt(tPtr,_v6Path.p->localSocket(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); _v6Path.p->sent(now); return true; } } else if (v4lr) { if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + attemptToContactAt(tPtr,_v4Path.p->localSocket(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); _v4Path.p->sent(now); return true; } @@ -416,13 +412,13 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) } else { if ( (inetAddressFamily == AF_INET) && ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + attemptToContactAt(tPtr,_v4Path.p->localSocket(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); _v4Path.p->sent(now); return true; } } else if ( (inetAddressFamily == AF_INET6) && ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + attemptToContactAt(tPtr,_v6Path.p->localSocket(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); _v6Path.p->sent(now); return true; } @@ -456,7 +452,6 @@ void Peer::writeState(void *tPtr,const uint64_t now) b.append(_v4Path.p->lastIn()); b.append(_v4Path.p->lastTrustEstablishedPacketReceived()); _v4Path.p->address().serialize(b); - _v4Path.p->localAddress().serialize(b); } if (_v6Path.lr) { b.append(_v6Path.lr); @@ -464,7 +459,6 @@ void Peer::writeState(void *tPtr,const uint64_t now) b.append(_v6Path.p->lastIn()); b.append(_v6Path.p->lastTrustEstablishedPacketReceived()); _v6Path.p->address().serialize(b); - _v6Path.p->localAddress().serialize(b); } } @@ -491,7 +485,7 @@ void Peer::writeState(void *tPtr,const uint64_t now) uint64_t tmp[2]; tmp[0] = _id.address().toInt(); tmp[1] = 0; - RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size()); + //RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size()); _lastWroteState = now; } catch ( ... ) {} // sanity check, should not be possible @@ -522,22 +516,19 @@ bool Peer::applyStateUpdate(const void *data,unsigned int len) const uint64_t lastOut = b.at(ptr); ptr += 8; const uint64_t lastIn = b.at(ptr); ptr += 8; const uint64_t lastTrustEstablishedPacketReceived = b.at(ptr); ptr += 8; - InetAddress addr,localAddr; + InetAddress addr; ptr += addr.deserialize(b,ptr); - ptr += localAddr.deserialize(b,ptr); - if (addr.ss_family == localAddr.ss_family) { - _PeerPath *p = (_PeerPath *)0; - switch(addr.ss_family) { - case AF_INET: p = &_v4Path; break; - case AF_INET6: p = &_v6Path; break; - } - if (p) { - if ( (!p->p) || ((p->p->address() != addr)||(p->p->localAddress() != localAddr)) ) { - p->p = RR->topology->getPath(localAddr,addr); - } - p->lr = lr; - p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); + _PeerPath *p = (_PeerPath *)0; + switch(addr.ss_family) { + case AF_INET: p = &_v4Path; break; + case AF_INET6: p = &_v6Path; break; + } + if (p) { + if ( (!p->p) || (p->p->address() != addr) ) { + p->p = RR->topology->getPath(-1,addr); } + p->lr = lr; + p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); } } } diff --git a/node/Peer.hpp b/node/Peer.hpp index f0eb3ee8..478c7232 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -154,12 +154,12 @@ public: * No statistics or sent times are updated here. * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param localAddr Local address + * @param localSocket Local source socket * @param atAddress Destination address * @param now Current time * @param counter Outgoing packet counter */ - void sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter); + void sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,unsigned int counter); /** * Send ECHO (or HELLO for older peers) to this peer at the given address @@ -167,13 +167,13 @@ public: * No statistics or sent times are updated here. * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param localAddr Local address + * @param localSocket Local source socket * @param atAddress Destination address * @param now Current time * @param sendFullHello If true, always send a full HELLO instead of just an ECHO * @param counter Outgoing packet counter */ - void attemptToContactAt(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); + void attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); /** * Try a memorized or statically defined path if any are known @@ -227,11 +227,11 @@ public: { Mutex::Lock _l(_paths_m); if ((inetAddressFamily == AF_INET)&&(_v4Path.lr)&&(_v4Path.p->address().ipScope() == scope)) { - attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + attemptToContactAt(tPtr,_v4Path.p->localSocket(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); _v4Path.p->sent(now); _v4Path.lr = 0; // path will not be used unless it speaks again } else if ((inetAddressFamily == AF_INET6)&&(_v6Path.lr)&&(_v6Path.p->address().ipScope() == scope)) { - attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + attemptToContactAt(tPtr,_v6Path.p->localSocket(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); _v6Path.p->sent(now); _v6Path.lr = 0; // path will not be used unless it speaks again } diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index ee0c8c24..99afe25d 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -67,6 +67,11 @@ public: Utils::burn(reinterpret_cast(const_cast(secretIdentityStr.data())),(unsigned int)secretIdentityStr.length()); } + /** + * A random integer identifying this running instance in a cluster + */ + uint64_t instanceId; + // Node instance that owns this RuntimeEnvironment Node *const node; @@ -90,11 +95,6 @@ public: Multicaster *mc; Topology *topology; SelfAwareness *sa; - - /** - * A random integer identifying this run of ZeroTier - */ - uint32_t instanceId; }; } // namespace ZeroTier diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index c5daddc3..3e3397f5 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -69,7 +69,7 @@ SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) : { } -void SelfAwareness::iam(void *tPtr,const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) +void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) { const InetAddress::IpScope scope = myPhysicalAddress.ipScope(); @@ -77,7 +77,7 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const InetAddress &re return; Mutex::Lock _l(_phy_m); - PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,receivedOnLocalAddress,reporterPhysicalAddress,scope)]; + PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,receivedOnLocalSocket,reporterPhysicalAddress,scope)]; if ( (trusted) && ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) { // Changes to external surface reported by trusted peers causes path reset in this scope diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index 63c416bf..35e0ad39 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -55,7 +55,7 @@ public: * @param trusted True if this peer is trusted as an authority to inform us of external address changes * @param now Current time */ - void iam(void *tPtr,const Address &reporter,const InetAddress &receivedOnLocalAddress,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); + void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); /** * Clean up database periodically @@ -75,15 +75,15 @@ private: struct PhySurfaceKey { Address reporter; - InetAddress receivedOnLocalAddress; + int64_t receivedOnLocalSocket; InetAddress reporterPhysicalAddress; InetAddress::IpScope scope; PhySurfaceKey() : reporter(),scope(InetAddress::IP_SCOPE_NONE) {} - PhySurfaceKey(const Address &r,const InetAddress &rol,const InetAddress &ra,InetAddress::IpScope s) : reporter(r),receivedOnLocalAddress(rol),reporterPhysicalAddress(ra),scope(s) {} + PhySurfaceKey(const Address &r,const int64_t rol,const InetAddress &ra,InetAddress::IpScope s) : reporter(r),receivedOnLocalSocket(rol),reporterPhysicalAddress(ra),scope(s) {} - inline unsigned long hashCode() const throw() { return ((unsigned long)reporter.toInt() + (unsigned long)scope); } - inline bool operator==(const PhySurfaceKey &k) const throw() { return ((reporter == k.reporter)&&(receivedOnLocalAddress == k.receivedOnLocalAddress)&&(reporterPhysicalAddress == k.reporterPhysicalAddress)&&(scope == k.scope)); } + inline unsigned long hashCode() const { return ((unsigned long)reporter.toInt() + (unsigned long)scope); } + inline bool operator==(const PhySurfaceKey &k) const { return ((reporter == k.reporter)&&(receivedOnLocalSocket == k.receivedOnLocalSocket)&&(reporterPhysicalAddress == k.reporterPhysicalAddress)&&(scope == k.scope)); } }; struct PhySurfaceEntry { diff --git a/node/Switch.cpp b/node/Switch.cpp index cbd73a83..a77ca89e 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -71,12 +71,12 @@ Switch::Switch(const RuntimeEnvironment *renv) : { } -void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len) +void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddress &fromAddr,const void *data,unsigned int len) { try { const uint64_t now = RR->node->now(); - SharedPtr path(RR->topology->getPath(localAddr,fromAddr)); + SharedPtr path(RR->topology->getPath(localSocket,fromAddr)); path->received(now); if (len == 13) { @@ -88,7 +88,7 @@ void Switch::onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAd const Address beaconAddr(reinterpret_cast(data) + 8,5); if (beaconAddr == RR->identity.address()) return; - if (!RR->node->shouldUsePathForZeroTierTraffic(tPtr,beaconAddr,localAddr,fromAddr)) + if (!RR->node->shouldUsePathForZeroTierTraffic(tPtr,beaconAddr,localSocket,fromAddr)) return; const SharedPtr peer(RR->topology->getPeer(tPtr,beaconAddr)); if (peer) { // we'll only respond to beacons from known peers @@ -752,7 +752,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) viaPath = peer->getBestPath(now,false); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(tPtr,viaPath->localAddress(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); + peer->attemptToContactAt(tPtr,viaPath->localSocket(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); viaPath->sent(now); } viaPath.zero(); diff --git a/node/Switch.hpp b/node/Switch.hpp index 9793dd45..cebe9e67 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -68,12 +68,12 @@ public: * Called when a packet is received from the real network * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param localAddr Local interface address + * @param localSocket Local I/O socket as supplied by external code * @param fromAddr Internet IP address of origin * @param data Packet data * @param len Packet length */ - void onRemotePacket(void *tPtr,const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len); + void onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddress &fromAddr,const void *data,unsigned int len); /** * Called when a packet comes from a local Ethernet tap diff --git a/node/Topology.cpp b/node/Topology.cpp index 09a1a895..d4632f43 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -125,10 +125,11 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) return *ap; } + /* try { char buf[ZT_PEER_MAX_SERIALIZED_STATE_SIZE]; uint64_t idbuf[2]; idbuf[0] = zta.toInt(); idbuf[1] = 0; - int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER_STATE,idbuf,buf,(unsigned int)sizeof(buf)); + int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER,idbuf,buf,(unsigned int)sizeof(buf)); if (len > 0) { Mutex::Lock _l(_peers_m); SharedPtr &ap = _peers[zta]; @@ -140,6 +141,7 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) return ap; } } catch ( ... ) {} // ignore invalid identities or other strage failures + */ return SharedPtr(); } diff --git a/node/Topology.hpp b/node/Topology.hpp index 32e38dd3..5f3e2da1 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -110,11 +110,11 @@ public: /** * Get a Path object for a given local and remote physical address, creating if needed * - * @param l Local address or NULL for 'any' or 'wildcard' + * @param l Local socket * @param r Remote address * @return Pointer to canonicalized Path object */ - inline SharedPtr getPath(const InetAddress &l,const InetAddress &r) + inline SharedPtr getPath(const int64_t l,const InetAddress &r) { Mutex::Lock _l(_paths_m); SharedPtr &p = _paths[Path::HashKey(l,r)]; diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index b1fe5921..040f3e46 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -88,11 +88,7 @@ class Binder : NonCopyable private: struct _Binding { - _Binding() : - udpSock((PhySocket *)0), - tcpListenSock((PhySocket *)0), - address() {} - + _Binding() : udpSock((PhySocket *)0),tcpListenSock((PhySocket *)0) {} PhySocket *udpSock; PhySocket *tcpListenSock; InetAddress address; @@ -373,93 +369,6 @@ public: _bindings.swap(newBindings); } - /** - * Send a UDP packet from the specified local interface, or all - * - * Unfortunately even by examining the routing table there is no ultimately - * robust way to tell where we might reach another host that works in all - * environments. As a result, we send packets with null (wildcard) local - * addresses from *every* bound interface. - * - * These are typically initial HELLOs, path probes, etc., since normal - * conversations will have a local endpoint address. So the cost is low and - * if the peer is not reachable via that route then the packet will go - * nowhere and nothing will happen. - * - * It will of course only send via interface bindings of the same socket - * family. No point in sending V4 via V6 or vice versa. - * - * In any case on most hosts there's only one or two interfaces that we - * will use, so none of this is particularly costly. - * - * @param local Local interface address or null address for 'all' - * @param remote Remote address - * @param data Data to send - * @param len Length of data - * @param v4ttl If non-zero, send this packet with the specified IP TTL (IPv4 only) - * @return -1 == local doesn't match any bound address, 0 == send failure, 1 == send successful - */ - template - inline int udpSend(Phy &phy,const InetAddress &local,const InetAddress &remote,const void *data,unsigned int len,unsigned int v4ttl = 0) const - { - PhySocket *s; - typename std::vector<_Binding>::const_iterator i; - int result; - Mutex::Lock _l(_lock); - - if (remote.ss_family == AF_INET) { - if (local) { - for(i=_bindings.begin();i!=_bindings.end();++i) { - if ( - (i->address.ss_family == AF_INET) && - (reinterpret_cast(&(i->address))->sin_port == reinterpret_cast(&local)->sin_port) && - (reinterpret_cast(&(i->address))->sin_addr.s_addr == reinterpret_cast(&local)->sin_addr.s_addr) - ) - { - s = i->udpSock; - goto Binder_send_packet; - } - } - } else { - for(i=_bindings.begin();i!=_bindings.end();++i) { - if (i->address.ss_family == AF_INET) { - s = i->udpSock; - goto Binder_send_packet; - } - } - } - } else { - if (local) { - for(i=_bindings.begin();i!=_bindings.end();++i) { - if ( - (i->address.ss_family == AF_INET6) && - (reinterpret_cast(&(i->address))->sin6_port == reinterpret_cast(&local)->sin6_port) && - (!memcmp(reinterpret_cast(&(i->address))->sin6_addr.s6_addr,reinterpret_cast(&local)->sin6_addr.s6_addr,16)) - ) - { - s = i->udpSock; - goto Binder_send_packet; - } - } - } else { - for(i=_bindings.begin();i!=_bindings.end();++i) { - if (i->address.ss_family == AF_INET6) { - s = i->udpSock; - goto Binder_send_packet; - } - } - } - } - - return -1; - -Binder_send_packet: - if (v4ttl) phy.setIp4UdpTtl(s,v4ttl); - result = (int)phy.udpSend(s,reinterpret_cast(&remote),data,len); - if (v4ttl) phy.setIp4UdpTtl(s,255); - return result; - } - /** * @return All currently bound local interface addresses */ @@ -472,6 +381,22 @@ Binder_send_packet: return aa; } + /** + * Send from all bound UDP sockets + */ + template + inline bool udpSendAll(Phy &phy,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) + { + bool r = false; + Mutex::Lock _l(_lock); + for(std::vector<_Binding>::const_iterator b(_bindings.begin());b!=_bindings.end();++b) { + if (ttl) phy.setIp4UdpTtl(b->udpSock,ttl); + if (phy.udpSend(b->udpSock,(const struct sockaddr *)addr,data,len)) r = true; + if (ttl) phy.setIp4UdpTtl(b->udpSock,255); + } + return r; + } + /** * @param addr Address to check * @return True if this is a bound local interface address diff --git a/service/OneService.cpp b/service/OneService.cpp index b5b11111..6497ae20 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -59,8 +59,6 @@ #include "../osdep/ManagedRoute.hpp" #include "OneService.hpp" -#include "ClusterGeoIpService.hpp" -#include "ClusterDefinition.hpp" #include "SoftwareUpdater.hpp" #ifdef __WINDOWS__ @@ -157,9 +155,6 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } // Maximum write buffer size for outgoing TCP connections (sanity limit) #define ZT_TCP_MAX_WRITEQ_SIZE 33554432 -// How often to check TCP connections and cluster links and send status to cluster peers -#define ZT_TCP_CHECK_PERIOD 15000 - // TCP activity timeout #define ZT_TCP_ACTIVITY_TIMEOUT 60000 @@ -311,9 +306,9 @@ static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData); static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len); static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen); -static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); +static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,int64_t localSocket,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl); static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr); +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int64_t localSocket,const struct sockaddr_storage *remoteAddr); static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result); static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len); @@ -362,8 +357,7 @@ struct TcpConnection TCP_UNCATEGORIZED_INCOMING, // uncategorized incoming connection TCP_HTTP_INCOMING, TCP_HTTP_OUTGOING, - TCP_TUNNEL_OUTGOING, // TUNNELED mode proxy outbound connection - TCP_CLUSTER_BACKPLANE + TCP_TUNNEL_OUTGOING // TUNNELED mode proxy outbound connection } type; OneServiceImpl *parent; @@ -380,29 +374,11 @@ struct TcpConnection std::string status; std::map< std::string,std::string > headers; - // Used for cluster backplane connections - uint64_t clusterMemberId; - unsigned int clusterMemberVersionMajor; - unsigned int clusterMemberVersionMinor; - unsigned int clusterMemberVersionRev; - std::vector< InetAddress > clusterMemberLocalAddresses; - Mutex clusterMemberLocalAddresses_m; - std::string readq; std::string writeq; Mutex writeq_m; }; -/** - * Message types for cluster backplane communication - */ -enum ClusterMessageType -{ - CLUSTER_MESSAGE_STATUS = 0, - CLUSTER_MESSAGE_STATE_OBJECT = 1, - CLUSTER_MESSAGE_PROXY_SEND = 2 -}; - class OneServiceImpl : public OneService { public: @@ -421,8 +397,6 @@ public: bool _updateAutoApply; unsigned int _primaryPort; volatile unsigned int _udpPortPickerCounter; - uint64_t _clusterMemberId; - uint8_t _clusterKey[32]; // secret key for cluster backplane config // Local configuration and memo-ized information from it json _localConfig; @@ -434,7 +408,6 @@ public: std::vector< InetAddress > _globalV6Blacklist; std::vector< InetAddress > _allowManagementFrom; std::vector< std::string > _interfacePrefixBlacklist; - std::vector< InetAddress > _clusterBackplaneAddresses; Mutex _localConfig_m; /* @@ -518,7 +491,6 @@ public: ,_updateAutoApply(false) ,_primaryPort(port) ,_udpPortPickerCounter(0) - ,_clusterMemberId(0) ,_lastDirectReceiveFromGlobal(0) #ifdef ZT_TCP_FALLBACK_RELAY ,_lastSendToGlobalV4(0) @@ -754,23 +726,6 @@ public: } } - // Derive the cluster's shared secret backplane encryption key by hashing its shared secret identity - { - uint8_t tmp[64]; - uint8_t sk[ZT_C25519_PRIVATE_KEY_LEN + 4]; - memcpy(sk,_node->identity().privateKeyPair().priv.data,ZT_C25519_PRIVATE_KEY_LEN); - sk[ZT_C25519_PRIVATE_KEY_LEN] = 0xab; - sk[ZT_C25519_PRIVATE_KEY_LEN + 1] = 0xcd; - sk[ZT_C25519_PRIVATE_KEY_LEN + 2] = 0xef; - sk[ZT_C25519_PRIVATE_KEY_LEN + 3] = 0xab; // add an arbitrary nonce, just because - SHA512::hash(tmp,sk,ZT_C25519_PRIVATE_KEY_LEN + 4); - memcpy(_clusterKey,tmp,32); - } - - // Assign a random non-zero cluster member ID to identify vs. other cluster members - Utils::getSecureRandom(&_clusterMemberId,sizeof(_clusterMemberId)); - if (!_clusterMemberId) _clusterMemberId = 1; - // Main I/O loop _nextBackgroundTaskDeadline = 0; uint64_t clockShouldBe = OSUtils::now(); @@ -779,7 +734,6 @@ public: uint64_t lastBindRefresh = 0; uint64_t lastUpdateCheck = clockShouldBe; uint64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle - uint64_t lastTcpCheck = 0; for(;;) { _run_m.lock(); if (!_run) { @@ -873,58 +827,6 @@ public: _node->addLocalInterfaceAddress(reinterpret_cast(&(*i))); } - // Check TCP connections and cluster links - if ((now - lastTcpCheck) >= ZT_TCP_CHECK_PERIOD) { - lastTcpCheck = now; - - // Send status to active cluster links and close overflowed and dead ones - std::vector toClose; - std::vector clusterLinksUp; - { - Mutex::Lock _l(_tcpConnections_m); - for(std::vector::const_iterator c(_tcpConnections.begin());c!=_tcpConnections.end();++c) { - TcpConnection *const tc = *c; - tc->writeq_m.lock(); - const unsigned long wql = (unsigned long)tc->writeq.length(); - tc->writeq_m.unlock(); - if ((tc->sock)&&((wql > ZT_TCP_MAX_WRITEQ_SIZE)||((now - tc->lastReceive) > ZT_TCP_ACTIVITY_TIMEOUT))) { - toClose.push_back(tc->sock); - } else if ((tc->type == TcpConnection::TCP_CLUSTER_BACKPLANE)&&(tc->clusterMemberId)) { - clusterLinksUp.push_back(tc->remoteAddr); - sendMyCurrentClusterState(tc); - } - } - } - for(std::vector::iterator s(toClose.begin());s!=toClose.end();++s) - _phy.close(*s,true); - - // Attempt to connect to cluster links we don't have an active connection to - { - Mutex::Lock _l(_localConfig_m); - for(std::vector::const_iterator ca(_clusterBackplaneAddresses.begin());ca!=_clusterBackplaneAddresses.end();++ca) { - if ( (std::find(clusterLinksUp.begin(),clusterLinksUp.end(),*ca) == clusterLinksUp.end()) && (!_binder.isBoundLocalInterfaceAddress(*ca)) ) { - TcpConnection *tc = new TcpConnection(); - { - Mutex::Lock _l(_tcpConnections_m); - _tcpConnections.push_back(tc); - } - - tc->type = TcpConnection::TCP_CLUSTER_BACKPLANE; - tc->remoteAddr = *ca; - tc->lastReceive = OSUtils::now(); - tc->parent = this; - tc->sock = (PhySocket *)0; // set in connect handler - tc->messageSize = 0; - - tc->clusterMemberId = 0; // not known yet - - bool connected = false; - _phy.tcpConnect(reinterpret_cast(&(*ca)),connected,(void *)tc,true); - } - } - } - } - const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 100; clockShouldBe = now + (uint64_t)delay; _phy.poll(delay); @@ -1211,21 +1113,6 @@ public: res["planetWorldId"] = planet.id(); res["planetWorldTimestamp"] = planet.timestamp(); - { - json cj(json::object()); - Mutex::Lock _l(_tcpConnections_m); - Mutex::Lock _l2(_localConfig_m); - for(std::vector::const_iterator ca(_clusterBackplaneAddresses.begin());ca!=_clusterBackplaneAddresses.end();++ca) { - uint64_t up = 0; - for(std::vector::const_iterator c(_tcpConnections.begin());c!=_tcpConnections.end();++c) { - if (((*c)->remoteAddr == *ca)&&((*c)->clusterMemberId)&&((*c)->lastReceive > up)) - up = (*c)->lastReceive; - } - cj[ca->toString()] = up; - } - res["cluster"] = cj; - } - scode = 200; } else if (ps[0] == "moon") { std::vector moons(_node->moons()); @@ -1576,16 +1463,6 @@ public: } } - json &cl = settings["cluster"]; - _clusterBackplaneAddresses.clear(); - if (cl.is_array()) { - for(unsigned long i=0;i buf; - - buf.appendRandom(16); - buf.addSize(8); // space for MAC - buf.append((uint8_t)CLUSTER_MESSAGE_STATUS); - buf.append(_clusterMemberId); - buf.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR); - buf.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR); - buf.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); - - std::vector lif(_binder.allBoundLocalInterfaceAddresses()); - buf.append((uint16_t)lif.size()); - for(std::vector::const_iterator i(lif.begin());i!=lif.end();++i) - i->serialize(buf); - - Mutex::Lock _l(tc->writeq_m); - - if (tc->writeq.length() == 0) - _phy.setNotifyWritable(tc->sock,true); - - const unsigned int mlen = buf.size(); - tc->writeq.push_back((char)((mlen >> 16) & 0xff)); - tc->writeq.push_back((char)((mlen >> 8) & 0xff)); - tc->writeq.push_back((char)(mlen & 0xff)); - - char *const data = reinterpret_cast(buf.unsafeData()); - encryptClusterMessage(data,mlen); - tc->writeq.append(data,mlen); - } catch ( ... ) { - fprintf(stderr,"WARNING: unexpected exception announcing status to cluster members" ZT_EOL_S); - } - } - - bool proxySendViaCluster(const InetAddress &fromAddress,const InetAddress &dest,const void *data,unsigned int len,unsigned int ttl) - { - Mutex::Lock _l(_tcpConnections_m); - for(std::vector::const_iterator c(_tcpConnections.begin());c!=_tcpConnections.end();++c) { - TcpConnection *const tc = *c; - if ((tc->type == TcpConnection::TCP_CLUSTER_BACKPLANE)&&(tc->clusterMemberId)) { - Mutex::Lock _l2(tc->clusterMemberLocalAddresses_m); - for(std::vector::const_iterator i(tc->clusterMemberLocalAddresses.begin());i!=tc->clusterMemberLocalAddresses.end();++i) { - if (*i == fromAddress) { - Buffer<1024> buf; - - buf.appendRandom(16); - buf.addSize(8); // space for MAC - buf.append((uint8_t)CLUSTER_MESSAGE_PROXY_SEND); - buf.append((uint8_t)ttl); - dest.serialize(buf); - fromAddress.serialize(buf); - - Mutex::Lock _l3(tc->writeq_m); - - if (tc->writeq.length() == 0) - _phy.setNotifyWritable(tc->sock,true); - - const unsigned int mlen = buf.size() + len; - tc->writeq.push_back((char)((mlen >> 16) & 0xff)); - tc->writeq.push_back((char)((mlen >> 8) & 0xff)); - tc->writeq.push_back((char)(mlen & 0xff)); - - const unsigned long startpos = (unsigned long)tc->writeq.length(); - tc->writeq.append(reinterpret_cast(buf.data()),buf.size()); - tc->writeq.append(reinterpret_cast(data),len); - - char *const outdata = const_cast(tc->writeq.data()) + startpos; - encryptClusterMessage(outdata,mlen); - - return true; - } - } - } - } - return false; - } - - void replicateStateObject(const ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len,TcpConnection *tc) - { - char buf[42]; - Mutex::Lock _l2(tc->writeq_m); - - if (tc->writeq.length() == 0) - _phy.setNotifyWritable(tc->sock,true); - - const unsigned int mlen = len + 42; - - tc->writeq.push_back((char)((mlen >> 16) & 0xff)); - tc->writeq.push_back((char)((mlen >> 8) & 0xff)); - tc->writeq.push_back((char)(mlen & 0xff)); - - Utils::getSecureRandom(buf,16); - buf[24] = (char)CLUSTER_MESSAGE_STATE_OBJECT; - buf[25] = (char)type; - buf[26] = (char)((id[0] >> 56) & 0xff); - buf[27] = (char)((id[0] >> 48) & 0xff); - buf[28] = (char)((id[0] >> 40) & 0xff); - buf[29] = (char)((id[0] >> 32) & 0xff); - buf[30] = (char)((id[0] >> 24) & 0xff); - buf[31] = (char)((id[0] >> 16) & 0xff); - buf[32] = (char)((id[0] >> 8) & 0xff); - buf[33] = (char)(id[0] & 0xff); - buf[34] = (char)((id[1] >> 56) & 0xff); - buf[35] = (char)((id[1] >> 48) & 0xff); - buf[36] = (char)((id[1] >> 40) & 0xff); - buf[37] = (char)((id[1] >> 32) & 0xff); - buf[38] = (char)((id[1] >> 24) & 0xff); - buf[39] = (char)((id[1] >> 16) & 0xff); - buf[40] = (char)((id[1] >> 8) & 0xff); - buf[41] = (char)(id[1] & 0xff); - - const unsigned long startpos = (unsigned long)tc->writeq.length(); - tc->writeq.append(buf,42); - tc->writeq.append(reinterpret_cast(data),len); - - char *const outdata = const_cast(tc->writeq.data()) + startpos; - encryptClusterMessage(outdata,mlen); - tc->writeq.append(outdata,mlen); - } - - void writeStateObject(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) - { - char buf[65535]; - char p[1024]; - FILE *f; - bool secure = false; - - switch(type) { - case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); - break; - case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); - secure = true; - break; - //case ZT_STATE_OBJECT_PEER_STATE: - // break; - case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); - secure = true; - break; - //case ZT_STATE_OBJECT_NETWORK_MEMBERSHIP: - // break; - case ZT_STATE_OBJECT_PLANET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); - break; - case ZT_STATE_OBJECT_MOON: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); - break; - default: - p[0] = (char)0; - break; - } - - if (p[0]) { - if (len >= 0) { - // Check to see if we've already written this first. This reduces - // redundant writes and I/O overhead on most platforms and has - // little effect on others. - f = fopen(p,"r"); - bool redundant = false; - if (f) { - long l = (long)fread(buf,1,sizeof(buf),f); - fclose(f); - redundant = ((l == (long)len)&&(memcmp(data,buf,l) == 0)); - } - if (!redundant) { - f = fopen(p,"w"); - if (f) { - if (fwrite(data,len,1,f) != 1) - fprintf(stderr,"WARNING: unable to write to file: %s (I/O error)" ZT_EOL_S,p); - fclose(f); - if (secure) - OSUtils::lockDownFile(p,false); - } else { - fprintf(stderr,"WARNING: unable to write to file: %s (unable to open)" ZT_EOL_S,p); - } - } - } else { - OSUtils::rm(p); - } - } - } - - void sendMyCurrentClusterState(TcpConnection *tc) - { - // We currently don't need to dump everything. Networks and moons are most important. - // The rest will get caught up rapidly due to constant peer updates, etc. - std::string buf; - std::vector l(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str(),false)); - for(std::vector::const_iterator f(l.begin());f!=l.end();++f) { - buf.clear(); - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + *f).c_str(),buf)) { - if (f->length() == 21) { - const uint64_t nwid = Utils::hexStrToU64(f->substr(0,16).c_str()); - if (nwid) { - uint64_t tmp[2]; - tmp[0] = nwid; - tmp[1] = 0; - replicateStateObject(ZT_STATE_OBJECT_NETWORK_CONFIG,tmp,buf.data(),(int)buf.length(),tc); - } - } - } - } - l = OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S + "moons.d").c_str(),false); - for(std::vector::const_iterator f(l.begin());f!=l.end();++f) { - buf.clear(); - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + *f).c_str(),buf)) { - if (f->length() == 21) { - const uint64_t moonId = Utils::hexStrToU64(f->substr(0,16).c_str()); - if (moonId) { - uint64_t tmp[2]; - tmp[0] = moonId; - tmp[1] = 0; - replicateStateObject(ZT_STATE_OBJECT_MOON,tmp,buf.data(),(int)buf.length(),tc); - } - } - } - } - } - // ========================================================================= // Handlers for Node and Phy<> callbacks // ========================================================================= @@ -2010,7 +1643,7 @@ public: const ZT_ResultCode rc = _node->processWirePacket( (void *)0, OSUtils::now(), - reinterpret_cast(localAddr), + (int64_t)((uintptr_t)sock), (const struct sockaddr_storage *)from, // Phy<> uses sockaddr_storage, so it'll always be that big data, len, @@ -2044,13 +1677,6 @@ public: _phy.close(_tcpFallbackTunnel->sock); _tcpFallbackTunnel = tc; _phy.streamSend(sock,ZT_TCP_TUNNEL_HELLO,sizeof(ZT_TCP_TUNNEL_HELLO)); - } else if (tc->type == TcpConnection::TCP_CLUSTER_BACKPLANE) { - { - Mutex::Lock _l(tc->writeq_m); - tc->writeq.push_back((char)0x93); // identifies type of connection as cluster backplane - } - announceStatusToClusterMember(tc); - _phy.setNotifyWritable(sock,true); } else { _phy.close(sock,true); } @@ -2106,31 +1732,6 @@ public: case TcpConnection::TCP_UNCATEGORIZED_INCOMING: switch(reinterpret_cast(data)[0]) { - // 0x93 is first byte of cluster backplane connections - case 0x93: { - // We only allow this from cluster backplane IPs. We also authenticate - // each packet cryptographically, so this is just a first line of defense. - bool allow = false; - { - Mutex::Lock _l(_localConfig_m); - for(std::vector< InetAddress >::const_iterator i(_clusterBackplaneAddresses.begin());i!=_clusterBackplaneAddresses.end();++i) { - if (tc->remoteAddr.ipsEqual(*i)) { - allow = true; - break; - } - } - } - if (allow) { - tc->type = TcpConnection::TCP_CLUSTER_BACKPLANE; - tc->clusterMemberId = 0; // unknown, waiting for first status message - announceStatusToClusterMember(tc); - if (len > 1) - phyOnTcpData(sock,uptr,reinterpret_cast(data) + 1,len - 1); - } else { - _phy.close(sock); - } - } break; - // HTTP: GET, PUT, POST, HEAD case 'G': case 'P': @@ -2223,7 +1824,7 @@ public: const ZT_ResultCode rc = _node->processWirePacket( (void *)0, OSUtils::now(), - reinterpret_cast(&fakeTcpLocalInterfaceAddress), + -1, reinterpret_cast(&from), data, plen, @@ -2248,114 +1849,6 @@ public: } return; - case TcpConnection::TCP_CLUSTER_BACKPLANE: - tc->readq.append((const char *)data,len); - if (tc->readq.length() >= 28) { // got 3-byte message size + 16-byte IV + 8-byte MAC + 1-byte type (encrypted) - uint8_t *data = reinterpret_cast(const_cast(tc->readq.data())); - unsigned long mlen = ( ((unsigned long)data[0] << 16) | ((unsigned long)data[1] << 8) | (unsigned long)data[2] ); - if ((mlen < 25)||(mlen > ZT_TCP_MAX_WRITEQ_SIZE)) { - _phy.close(sock); - return; - } else if (tc->readq.length() >= (mlen + 3)) { // got entire message - data += 3; - - uint8_t key[32]; - memcpy(key,_clusterKey,32); - for(int i=0;i<8;++i) key[i] ^= data[i]; // first 8 bytes of IV get XORed with key - Salsa20 s20(key,data + 8); // last 8 bytes of IV are fed into Salsa20 directly as its 64-bit IV - - uint8_t macKey[32]; - uint8_t mac[16]; - memset(macKey,0,32); - s20.crypt12(macKey,macKey,32); - Poly1305::compute(mac,data + 24,mlen - 24,macKey); - if (!Utils::secureEq(mac,data + 16,8)) { - _phy.close(sock); - return; - } - s20.crypt12(data + 24,data + 24,mlen - 24); - - switch((ClusterMessageType)data[24]) { - case CLUSTER_MESSAGE_STATUS: - if (mlen > (25 + 16)) { - Buffer<4096> tmp(data + 25,mlen - 25); - try { - const uint64_t cmid = tmp.at(0); - if (cmid == _clusterMemberId) { // shouldn't happen, but don't allow self-to-self - _phy.close(sock); - return; - } - if (!tc->clusterMemberId) { - tc->clusterMemberId = cmid; - sendMyCurrentClusterState(tc); - } - tc->clusterMemberVersionMajor = tmp.at(8); - tc->clusterMemberVersionMinor = tmp.at(10); - tc->clusterMemberVersionRev = tmp.at(12); - const unsigned int clusterMemberLocalAddressCount = tmp.at(14); - std::vector la; - unsigned int ptr = 16; - for(unsigned int k=0;kclusterMemberLocalAddresses_m); - tc->clusterMemberLocalAddresses.swap(la); - } - } catch ( ... ) {} - } - break; - - case CLUSTER_MESSAGE_STATE_OBJECT: - if (mlen > 42) { // type + object ID + [data] - uint64_t objId[2]; - objId[0] = ( - ((uint64_t)data[26] << 56) | - ((uint64_t)data[27] << 48) | - ((uint64_t)data[28] << 40) | - ((uint64_t)data[29] << 32) | - ((uint64_t)data[30] << 24) | - ((uint64_t)data[31] << 16) | - ((uint64_t)data[32] << 8) | - (uint64_t)data[33] - ); - objId[1] = ( - ((uint64_t)data[34] << 56) | - ((uint64_t)data[35] << 48) | - ((uint64_t)data[36] << 40) | - ((uint64_t)data[37] << 32) | - ((uint64_t)data[38] << 24) | - ((uint64_t)data[39] << 16) | - ((uint64_t)data[40] << 8) | - (uint64_t)data[41] - ); - if (_node->processStateUpdate((void *)0,(ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42)) == ZT_RESULT_OK) - writeStateObject((ZT_StateObjectType)data[25],objId,data + 42,(unsigned int)(mlen - 42)); - } - break; - - case CLUSTER_MESSAGE_PROXY_SEND: - if (mlen > 25) { - Buffer<4096> tmp(data + 25,mlen - 25); - try { - InetAddress dest,src; - const unsigned int ttl = (unsigned int)tmp[0]; - unsigned int ptr = 1; - ptr += dest.deserialize(tmp); - ptr += src.deserialize(tmp,ptr); - if (ptr < tmp.size()) - _binder.udpSend(_phy,src,dest,reinterpret_cast(tmp.data()) + ptr,tmp.size() - ptr,ttl); - } catch ( ... ) {} - } - break; - } - - tc->readq.erase(tc->readq.begin(),tc->readq.begin() + mlen); - } - } - return; - } } catch ( ... ) { _phy.close(sock); @@ -2549,18 +2042,57 @@ public: inline void nodeStatePutFunction(enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len) { - writeStateObject(type,id,data,len); + char p[1024]; + FILE *f; + bool secure = false; - std::vector sentTo; - { - Mutex::Lock _l(_tcpConnections_m); - for(std::vector::const_iterator ci(_tcpConnections.begin());ci!=_tcpConnections.end();++ci) { - TcpConnection *const c = *ci; - if ((c->type == TcpConnection::TCP_CLUSTER_BACKPLANE)&&(c->clusterMemberId != 0)&&(std::find(sentTo.begin(),sentTo.end(),c->clusterMemberId) == sentTo.end())) { - sentTo.push_back(c->clusterMemberId); - replicateStateObject(type,id,data,len,c); - } + switch(type) { + case ZT_STATE_OBJECT_IDENTITY_PUBLIC: + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + break; + case ZT_STATE_OBJECT_IDENTITY_SECRET: + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + secure = true; + break; + case ZT_STATE_OBJECT_PLANET: + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + break; + case ZT_STATE_OBJECT_MOON: + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); + break; + case ZT_STATE_OBJECT_NETWORK_CONFIG: + Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); + secure = true; + break; + default: + return; + } + + if (len >= 0) { + // Check to see if we've already written this first. This reduces + // redundant writes and I/O overhead on most platforms and has + // little effect on others. + f = fopen(p,"r"); + if (f) { + char buf[65535]; + long l = (long)fread(buf,1,sizeof(buf),f); + fclose(f); + if ((l == (long)len)&&(memcmp(data,buf,l) == 0)) + return; } + + f = fopen(p,"w"); + if (f) { + if (fwrite(data,len,1,f) != 1) + fprintf(stderr,"WARNING: unable to write to file: %s (I/O error)" ZT_EOL_S,p); + fclose(f); + if (secure) + OSUtils::lockDownFile(p,false); + } else { + fprintf(stderr,"WARNING: unable to write to file: %s (unable to open)" ZT_EOL_S,p); + } + } else { + OSUtils::rm(p); } } @@ -2596,7 +2128,7 @@ public: return -1; } - inline int nodeWirePacketSendFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) + inline int nodeWirePacketSendFunction(const int64_t localSocket,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) { #ifdef ZT_TCP_FALLBACK_RELAY if (addr->ss_family == AF_INET) { @@ -2646,20 +2178,13 @@ public: // proxy fallback, which is slow. #endif // ZT_TCP_FALLBACK_RELAY - switch (_binder.udpSend(_phy,*(reinterpret_cast(localAddr)),*(reinterpret_cast(addr)),data,len,ttl)) { - case -1: // local bound address not found, so see if a cluster peer owns it - if (localAddr->ss_family != 0) { - return (proxySendViaCluster(*(reinterpret_cast(localAddr)),*(reinterpret_cast(addr)),data,len,ttl)) ? 0 : -1; - } else { - return -1; // failure - } - break; - - case 0: // failure - return -1; - - default: // success - return 0; + if ((localSocket != 0)&&(localSocket != -1)) { + if ((ttl)&&(addr->ss_family == AF_INET)) _phy.setIp4UdpTtl((PhySocket *)((uintptr_t)localSocket),ttl); + const bool r = _phy.udpSend((PhySocket *)((uintptr_t)localSocket),(const struct sockaddr *)addr,data,len); + if ((ttl)&&(addr->ss_family == AF_INET)) _phy.setIp4UdpTtl((PhySocket *)((uintptr_t)localSocket),255); + return ((r) ? 0 : -1); + } else { + return ((_binder.udpSendAll(_phy,addr,data,len,ttl)) ? 0 : -1); } } @@ -2671,7 +2196,7 @@ public: n->tap->put(MAC(sourceMac),MAC(destMac),etherType,data,len); } - inline int nodePathCheckFunction(uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) + inline int nodePathCheckFunction(uint64_t ztaddr,const int64_t localSocket,const struct sockaddr_storage *remoteAddr) { // Make sure we're not trying to do ZeroTier-over-ZeroTier { @@ -2882,12 +2407,12 @@ static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_St { reinterpret_cast(uptr)->nodeStatePutFunction(type,id,data,len); } static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen) { return reinterpret_cast(uptr)->nodeStateGetFunction(type,id,data,maxlen); } -static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) -{ return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); } +static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,int64_t localSocket,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl) +{ return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localSocket,addr,data,len,ttl); } static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) { reinterpret_cast(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); } -static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr) -{ return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr,localAddr,remoteAddr); } +static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int64_t localSocket,const struct sockaddr_storage *remoteAddr) +{ return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr,localSocket,remoteAddr); } static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result) { return reinterpret_cast(uptr)->nodePathLookupFunction(ztaddr,family,result); } static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) -- cgit v1.2.3 From 640ad577d1f52140adbe42e87b2da931bf15f430 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 11:56:46 -0700 Subject: . --- attic/ClusterGeoIpService.cpp | 243 ++++++++++++++++++++++++++++++++++++++++ attic/ClusterGeoIpService.hpp | 151 +++++++++++++++++++++++++ make-bsd.mk | 5 - make-linux.mk | 4 - make-mac.mk | 4 - node/Constants.hpp | 5 - node/Path.hpp | 12 -- node/Peer.cpp | 157 -------------------------- node/Peer.hpp | 31 ----- node/Topology.cpp | 5 +- objects.mk | 1 - service/ClusterGeoIpService.cpp | 243 ---------------------------------------- service/ClusterGeoIpService.hpp | 151 ------------------------- 13 files changed, 395 insertions(+), 617 deletions(-) create mode 100644 attic/ClusterGeoIpService.cpp create mode 100644 attic/ClusterGeoIpService.hpp delete mode 100644 service/ClusterGeoIpService.cpp delete mode 100644 service/ClusterGeoIpService.hpp (limited to 'node') diff --git a/attic/ClusterGeoIpService.cpp b/attic/ClusterGeoIpService.cpp new file mode 100644 index 00000000..2dcc9179 --- /dev/null +++ b/attic/ClusterGeoIpService.cpp @@ -0,0 +1,243 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_ENABLE_CLUSTER + +#include + +#include + +#include "ClusterGeoIpService.hpp" + +#include "../node/Utils.hpp" +#include "../osdep/OSUtils.hpp" + +#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 + +namespace ZeroTier { + +ClusterGeoIpService::ClusterGeoIpService() : + _pathToCsv(), + _ipStartColumn(-1), + _ipEndColumn(-1), + _latitudeColumn(-1), + _longitudeColumn(-1), + _lastFileCheckTime(0), + _csvModificationTime(0), + _csvFileSize(0) +{ +} + +ClusterGeoIpService::~ClusterGeoIpService() +{ +} + +bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) +{ + Mutex::Lock _l(_lock); + + if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { + _lastFileCheckTime = OSUtils::now(); + if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) + _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); + } + + /* We search by looking up the upper bound of the sorted vXdb vectors + * and then iterating down for a matching IP range. We stop when we hit + * the beginning or an entry whose start and end are before the IP we + * are searching. */ + + if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { + _V4E key; + key.start = Utils::ntoh((uint32_t)(reinterpret_cast(&ip)->sin_addr.s_addr)); + std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); + while (i != _v4db.begin()) { + --i; + if ((key.start >= i->start)&&(key.start <= i->end)) { + x = i->x; + y = i->y; + z = i->z; + //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); + return true; + } else if ((key.start > i->start)&&(key.start > i->end)) + break; + } + } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { + _V6E key; + memcpy(key.start,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); + while (i != _v6db.begin()) { + --i; + const int s_vs_s = memcmp(key.start,i->start,16); + const int s_vs_e = memcmp(key.start,i->end,16); + if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { + x = i->x; + y = i->y; + z = i->z; + //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); + return true; + } else if ((s_vs_s > 0)&&(s_vs_e > 0)) + break; + } + } + + return false; +} + +void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) +{ + std::vector ls(OSUtils::split(line,",\t","\\","\"'")); + if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& + ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& + ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& + ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { + InetAddress ipStart(ls[ipStartColumn].c_str(),0); + InetAddress ipEnd(ls[ipEndColumn].c_str(),0); + const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); + const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); + + if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { + const double latRadians = lat * 0.01745329251994; // PI / 180 + const double lonRadians = lon * 0.01745329251994; // PI / 180 + const double cosLat = cos(latRadians); + const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers + const int y = (int)round(6371.0 * sin(latRadians)); + const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); + + if (ipStart.ss_family == AF_INET) { + v4db.push_back(_V4E()); + v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast(&ipStart)->sin_addr.s_addr)); + v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast(&ipEnd)->sin_addr.s_addr)); + v4db.back().lat = (float)lat; + v4db.back().lon = (float)lon; + v4db.back().x = x; + v4db.back().y = y; + v4db.back().z = z; + //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); + } else if (ipStart.ss_family == AF_INET6) { + v6db.push_back(_V6E()); + memcpy(v6db.back().start,reinterpret_cast(&ipStart)->sin6_addr.s6_addr,16); + memcpy(v6db.back().end,reinterpret_cast(&ipEnd)->sin6_addr.s6_addr,16); + v6db.back().lat = (float)lat; + v6db.back().lon = (float)lon; + v6db.back().x = x; + v6db.back().y = y; + v6db.back().z = z; + //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); + } + } + } +} + +long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) +{ + // assumes _lock is locked + + FILE *f = fopen(pathToCsv,"rb"); + if (!f) + return -1; + + std::vector<_V4E> v4db; + std::vector<_V6E> v6db; + v4db.reserve(16777216); + v6db.reserve(16777216); + + char buf[4096]; + char linebuf[1024]; + unsigned int lineptr = 0; + for(;;) { + int n = (int)fread(buf,1,sizeof(buf),f); + if (n <= 0) + break; + for(int i=0;i 0)||(v6db.size() > 0)) { + std::sort(v4db.begin(),v4db.end()); + std::sort(v6db.begin(),v6db.end()); + + _pathToCsv = pathToCsv; + _ipStartColumn = ipStartColumn; + _ipEndColumn = ipEndColumn; + _latitudeColumn = latitudeColumn; + _longitudeColumn = longitudeColumn; + + _lastFileCheckTime = OSUtils::now(); + _csvModificationTime = OSUtils::getLastModified(pathToCsv); + _csvFileSize = OSUtils::getFileSize(pathToCsv); + + _v4db.swap(v4db); + _v6db.swap(v6db); + + return (long)(_v4db.size() + _v6db.size()); + } else { + return 0; + } +} + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +/* +int main(int argc,char **argv) +{ + char buf[1024]; + + ZeroTier::ClusterGeoIpService gip; + printf("loading...\n"); + gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); + printf("... done!\n"); fflush(stdout); + + while (gets(buf)) { // unsafe, testing only + ZeroTier::InetAddress addr(buf,0); + printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); + int x = 0,y = 0,z = 0; + if (gip.locate(addr,x,y,z)) { + //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); + } else { + printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); + } + } + + return 0; +} +*/ diff --git a/attic/ClusterGeoIpService.hpp b/attic/ClusterGeoIpService.hpp new file mode 100644 index 00000000..380f944f --- /dev/null +++ b/attic/ClusterGeoIpService.hpp @@ -0,0 +1,151 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CLUSTERGEOIPSERVICE_HPP +#define ZT_CLUSTERGEOIPSERVICE_HPP + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include +#include +#include + +#include +#include +#include + +#include "../node/Constants.hpp" +#include "../node/Mutex.hpp" +#include "../node/NonCopyable.hpp" +#include "../node/InetAddress.hpp" + +namespace ZeroTier { + +/** + * Loads a GeoIP CSV into memory for fast lookup, reloading as needed + * + * This was designed around the CSV from https://db-ip.com but can be used + * with any similar GeoIP CSV database that is presented in the form of an + * IP range and lat/long coordinates. + * + * It loads the whole database into memory, which can be kind of large. If + * the CSV file changes, the changes are loaded automatically. + */ +class ClusterGeoIpService : NonCopyable +{ +public: + ClusterGeoIpService(); + ~ClusterGeoIpService(); + + /** + * Load or reload CSV file + * + * CSV column indexes start at zero. CSVs can be quoted with single or + * double quotes. Whitespace before or after commas is ignored. Backslash + * may be used for escaping whitespace as well. + * + * @param pathToCsv Path to (uncompressed) CSV file + * @param ipStartColumn Column with IP range start + * @param ipEndColumn Column with IP range end (inclusive) + * @param latitudeColumn Column with latitude + * @param longitudeColumn Column with longitude + * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) + */ + inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) + { + Mutex::Lock _l(_lock); + return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); + } + + /** + * Attempt to locate an IP + * + * This returns true if x, y, and z are set. If the return value is false + * the values of x, y, and z are undefined. + * + * @param ip IPv4 or IPv6 address + * @param x Reference to variable to receive X + * @param y Reference to variable to receive Y + * @param z Reference to variable to receive Z + * @return True if coordinates were set + */ + bool locate(const InetAddress &ip,int &x,int &y,int &z); + + /** + * @return True if IP database/service is available for queries (otherwise locate() will always be false) + */ + inline bool available() const + { + Mutex::Lock _l(_lock); + return ((_v4db.size() + _v6db.size()) > 0); + } + +private: + struct _V4E + { + uint32_t start; + uint32_t end; + float lat,lon; + int16_t x,y,z; + + inline bool operator<(const _V4E &e) const { return (start < e.start); } + }; + + struct _V6E + { + uint8_t start[16]; + uint8_t end[16]; + float lat,lon; + int16_t x,y,z; + + inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } + }; + + static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); + long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); + + std::string _pathToCsv; + int _ipStartColumn; + int _ipEndColumn; + int _latitudeColumn; + int _longitudeColumn; + + uint64_t _lastFileCheckTime; + uint64_t _csvModificationTime; + int64_t _csvFileSize; + + std::vector<_V4E> _v4db; + std::vector<_V6E> _v6db; + + Mutex _lock; +}; + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +#endif diff --git a/make-bsd.mk b/make-bsd.mk index cbab5810..c2fd6062 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -7,11 +7,6 @@ LIBS= include objects.mk ONE_OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o -# Build with ZT_ENABLE_CLUSTER=1 to build with cluster support -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - # "make debug" is a shortcut for this ifeq ($(ZT_DEBUG),1) CFLAGS+=-Wall -Werror -g -pthread $(INCLUDES) $(DEFS) diff --git a/make-linux.mk b/make-linux.mk index c27c600d..7017d31e 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -38,10 +38,6 @@ endif # Trying to use dynamically linked libhttp-parser causes tons of compatibility problems. ONE_OBJS+=ext/http-parser/http_parser.o -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - ifeq ($(ZT_SYNOLOGY), 1) DEFS+=-D__SYNOLOGY__ endif diff --git a/make-mac.mk b/make-mac.mk index 5622a41b..196b17cb 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,10 +33,6 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - # Use fast ASM Salsa20/12 for x64 processors DEFS+=-DZT_USE_X64_ASM_SALSA2012 CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o diff --git a/node/Constants.hpp b/node/Constants.hpp index 274b9564..12bf8d28 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -218,11 +218,6 @@ */ #define ZT_HOUSEKEEPING_PERIOD 60000 -/** - * How often in ms to write peer state to storage and/or cluster (approximate) - */ -#define ZT_PEER_STATE_WRITE_PERIOD 10000 - /** * How long to remember peer records in RAM if they haven't been used */ diff --git a/node/Path.hpp b/node/Path.hpp index 854b28e2..ac8e4c0e 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -283,18 +283,6 @@ public: */ inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } - /** - * @param lo Last out send - * @param li Last in send - * @param lt Last trust established packet received - */ - inline void updateFromRemoteState(const uint64_t lo,const uint64_t li,const uint64_t lt) - { - _lastOut = lo; - _lastIn = li; - _lastTrustEstablishedPacketReceived = lt; - } - /** * Return and increment outgoing packet counter (used with Packet::armor()) * diff --git a/node/Peer.cpp b/node/Peer.cpp index 875d651e..fb9a72b1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,8 +38,6 @@ namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : RR(renv), - _lastWroteState(0), - _lastReceivedStateTimestamp(0), _lastReceive(0), _lastNontrivialReceive(0), _lastTriedMemorizedPath(0), @@ -184,7 +182,6 @@ void Peer::received( if (verb == Packet::VERB_OK) { potentialNewPeerPath->lr = now; potentialNewPeerPath->p = path; - _lastWroteState = 0; // force state write now } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); @@ -263,9 +260,6 @@ void Peer::received( } } } - - if ((now - _lastWroteState) > ZT_PEER_STATE_WRITE_PERIOD) - writeState(tPtr,now); } bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) @@ -428,155 +422,4 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } -void Peer::writeState(void *tPtr,const uint64_t now) -{ - try { - Buffer b; - - b.append((uint8_t)1); // version - b.append(now); - - _id.serialize(b); - - { - Mutex::Lock _l(_paths_m); - unsigned int count = 0; - if (_v4Path.lr) - ++count; - if (_v6Path.lr) - ++count; - b.append((uint8_t)count); - if (_v4Path.lr) { - b.append(_v4Path.lr); - b.append(_v4Path.p->lastOut()); - b.append(_v4Path.p->lastIn()); - b.append(_v4Path.p->lastTrustEstablishedPacketReceived()); - _v4Path.p->address().serialize(b); - } - if (_v6Path.lr) { - b.append(_v6Path.lr); - b.append(_v6Path.p->lastOut()); - b.append(_v6Path.p->lastIn()); - b.append(_v6Path.p->lastTrustEstablishedPacketReceived()); - _v6Path.p->address().serialize(b); - } - } - - // Save space by sending these as time since now at 100ms resolution - b.append((uint16_t)(std::max(now - _lastReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastNontrivialReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastTriedMemorizedPath,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastDirectPathPushSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastDirectPathPushReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastCredentialRequestSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastWhoisRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastEchoRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastComRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastComRequestSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastCredentialsReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastTrustEstablishedPacketReceived,(uint64_t)6553500) / 100)); - - b.append((uint8_t)_vProto); - b.append((uint8_t)_vMajor); - b.append((uint8_t)_vMinor); - b.append((uint16_t)_vRevision); - - b.append((uint16_t)0); // length of additional fields - - uint64_t tmp[2]; - tmp[0] = _id.address().toInt(); tmp[1] = 0; - //RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size()); - - _lastWroteState = now; - } catch ( ... ) {} // sanity check, should not be possible -} - -bool Peer::applyStateUpdate(const void *data,unsigned int len) -{ - try { - Buffer b(data,len); - unsigned int ptr = 0; - - if (b[ptr++] != 1) - return false; - const uint64_t ts = b.at(ptr); ptr += 8; - if (ts <= _lastReceivedStateTimestamp) - return false; - - Identity id; - ptr += id.deserialize(b,ptr); - if (id != _id) // sanity check - return false; - - const unsigned int pathCount = (unsigned int)b[ptr++]; - { - Mutex::Lock _l(_paths_m); - for(unsigned int i=0;i(ptr); ptr += 8; - const uint64_t lastOut = b.at(ptr); ptr += 8; - const uint64_t lastIn = b.at(ptr); ptr += 8; - const uint64_t lastTrustEstablishedPacketReceived = b.at(ptr); ptr += 8; - InetAddress addr; - ptr += addr.deserialize(b,ptr); - _PeerPath *p = (_PeerPath *)0; - switch(addr.ss_family) { - case AF_INET: p = &_v4Path; break; - case AF_INET6: p = &_v6Path; break; - } - if (p) { - if ( (!p->p) || (p->p->address() != addr) ) { - p->p = RR->topology->getPath(-1,addr); - } - p->lr = lr; - p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); - } - } - } - - _lastReceive = std::max(_lastReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastNontrivialReceive = std::max(_lastNontrivialReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastDirectPathPushSent = std::max(_lastDirectPathPushSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastCredentialRequestSent = std::max(_lastCredentialRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastEchoRequestReceived = std::max(_lastEchoRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastComRequestReceived = std::max(_lastComRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastComRequestSent = std::max(_lastComRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastCredentialsReceived = std::max(_lastCredentialsReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - - _vProto = (uint16_t)b[ptr++]; - _vMajor = (uint16_t)b[ptr++]; - _vMinor = (uint16_t)b[ptr++]; - _vRevision = b.at(ptr); ptr += 2; - - _lastReceivedStateTimestamp = ts; - - return true; - } catch ( ... ) {} // ignore invalid state updates - return false; -} - -SharedPtr Peer::createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len) -{ - try { - Identity id; - { - Buffer b(data,len); - unsigned int ptr = 0; - if (b[ptr++] != 1) - return SharedPtr(); - ptr += 8; // skip TS, don't care - id.deserialize(b,ptr); - } - if (id) { - const SharedPtr p(new Peer(renv,renv->identity,id)); - if (p->applyStateUpdate(data,len)) - return renv->topology->addPeer(tPtr,p); - } - } catch ( ... ) {} - return SharedPtr(); -} - } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 478c7232..ad2d0ddc 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -195,23 +195,6 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); - /** - * Write object state to external storage and/or cluster network - * - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param now Current time - */ - void writeState(void *tPtr,const uint64_t now); - - /** - * Apply a state update received from e.g. a remote cluster member - * - * @param data State update data - * @param len Length of state update - * @return True if state update was applied, false if ignored or invalid - */ - bool applyStateUpdate(const void *data,unsigned int len); - /** * Reset paths within a given IP scope and address family * @@ -440,17 +423,6 @@ public: return false; } - /** - * Create a peer from a remote state update - * - * @param renv Runtime environment - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param data State update data - * @param len State update length - * @return Peer or NULL if data was invalid - */ - static SharedPtr createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len); - private: struct _PeerPath { @@ -463,9 +435,6 @@ private: const RuntimeEnvironment *RR; - uint64_t _lastWroteState; - uint64_t _lastReceivedStateTimestamp; - uint64_t _lastReceive; // direct or indirect uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. uint64_t _lastTriedMemorizedPath; diff --git a/node/Topology.cpp b/node/Topology.cpp index d4632f43..809bc7e7 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -395,11 +395,8 @@ void Topology::doPeriodicTasks(void *tPtr,uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) { + if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) _peers.erase(*a); - } else { - (*p)->writeState(tPtr,now); - } } } diff --git a/objects.mk b/objects.mk index c8231f08..3a8bd645 100644 --- a/objects.mk +++ b/objects.mk @@ -31,7 +31,6 @@ ONE_OBJS=\ osdep/ManagedRoute.o \ osdep/Http.o \ osdep/OSUtils.o \ - service/ClusterGeoIpService.o \ service/SoftwareUpdater.o \ service/OneService.o diff --git a/service/ClusterGeoIpService.cpp b/service/ClusterGeoIpService.cpp deleted file mode 100644 index 2dcc9179..00000000 --- a/service/ClusterGeoIpService.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifdef ZT_ENABLE_CLUSTER - -#include - -#include - -#include "ClusterGeoIpService.hpp" - -#include "../node/Utils.hpp" -#include "../osdep/OSUtils.hpp" - -#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 - -namespace ZeroTier { - -ClusterGeoIpService::ClusterGeoIpService() : - _pathToCsv(), - _ipStartColumn(-1), - _ipEndColumn(-1), - _latitudeColumn(-1), - _longitudeColumn(-1), - _lastFileCheckTime(0), - _csvModificationTime(0), - _csvFileSize(0) -{ -} - -ClusterGeoIpService::~ClusterGeoIpService() -{ -} - -bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) -{ - Mutex::Lock _l(_lock); - - if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { - _lastFileCheckTime = OSUtils::now(); - if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) - _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); - } - - /* We search by looking up the upper bound of the sorted vXdb vectors - * and then iterating down for a matching IP range. We stop when we hit - * the beginning or an entry whose start and end are before the IP we - * are searching. */ - - if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { - _V4E key; - key.start = Utils::ntoh((uint32_t)(reinterpret_cast(&ip)->sin_addr.s_addr)); - std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); - while (i != _v4db.begin()) { - --i; - if ((key.start >= i->start)&&(key.start <= i->end)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((key.start > i->start)&&(key.start > i->end)) - break; - } - } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { - _V6E key; - memcpy(key.start,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); - while (i != _v6db.begin()) { - --i; - const int s_vs_s = memcmp(key.start,i->start,16); - const int s_vs_e = memcmp(key.start,i->end,16); - if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((s_vs_s > 0)&&(s_vs_e > 0)) - break; - } - } - - return false; -} - -void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - std::vector ls(OSUtils::split(line,",\t","\\","\"'")); - if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& - ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& - ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& - ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { - InetAddress ipStart(ls[ipStartColumn].c_str(),0); - InetAddress ipEnd(ls[ipEndColumn].c_str(),0); - const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); - const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); - - if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { - const double latRadians = lat * 0.01745329251994; // PI / 180 - const double lonRadians = lon * 0.01745329251994; // PI / 180 - const double cosLat = cos(latRadians); - const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers - const int y = (int)round(6371.0 * sin(latRadians)); - const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); - - if (ipStart.ss_family == AF_INET) { - v4db.push_back(_V4E()); - v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast(&ipStart)->sin_addr.s_addr)); - v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast(&ipEnd)->sin_addr.s_addr)); - v4db.back().lat = (float)lat; - v4db.back().lon = (float)lon; - v4db.back().x = x; - v4db.back().y = y; - v4db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } else if (ipStart.ss_family == AF_INET6) { - v6db.push_back(_V6E()); - memcpy(v6db.back().start,reinterpret_cast(&ipStart)->sin6_addr.s6_addr,16); - memcpy(v6db.back().end,reinterpret_cast(&ipEnd)->sin6_addr.s6_addr,16); - v6db.back().lat = (float)lat; - v6db.back().lon = (float)lon; - v6db.back().x = x; - v6db.back().y = y; - v6db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } - } - } -} - -long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - // assumes _lock is locked - - FILE *f = fopen(pathToCsv,"rb"); - if (!f) - return -1; - - std::vector<_V4E> v4db; - std::vector<_V6E> v6db; - v4db.reserve(16777216); - v6db.reserve(16777216); - - char buf[4096]; - char linebuf[1024]; - unsigned int lineptr = 0; - for(;;) { - int n = (int)fread(buf,1,sizeof(buf),f); - if (n <= 0) - break; - for(int i=0;i 0)||(v6db.size() > 0)) { - std::sort(v4db.begin(),v4db.end()); - std::sort(v6db.begin(),v6db.end()); - - _pathToCsv = pathToCsv; - _ipStartColumn = ipStartColumn; - _ipEndColumn = ipEndColumn; - _latitudeColumn = latitudeColumn; - _longitudeColumn = longitudeColumn; - - _lastFileCheckTime = OSUtils::now(); - _csvModificationTime = OSUtils::getLastModified(pathToCsv); - _csvFileSize = OSUtils::getFileSize(pathToCsv); - - _v4db.swap(v4db); - _v6db.swap(v6db); - - return (long)(_v4db.size() + _v6db.size()); - } else { - return 0; - } -} - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -/* -int main(int argc,char **argv) -{ - char buf[1024]; - - ZeroTier::ClusterGeoIpService gip; - printf("loading...\n"); - gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); - printf("... done!\n"); fflush(stdout); - - while (gets(buf)) { // unsafe, testing only - ZeroTier::InetAddress addr(buf,0); - printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); - int x = 0,y = 0,z = 0; - if (gip.locate(addr,x,y,z)) { - //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); - } else { - printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); - } - } - - return 0; -} -*/ diff --git a/service/ClusterGeoIpService.hpp b/service/ClusterGeoIpService.hpp deleted file mode 100644 index 380f944f..00000000 --- a/service/ClusterGeoIpService.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTERGEOIPSERVICE_HPP -#define ZT_CLUSTERGEOIPSERVICE_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include -#include -#include -#include - -#include -#include -#include - -#include "../node/Constants.hpp" -#include "../node/Mutex.hpp" -#include "../node/NonCopyable.hpp" -#include "../node/InetAddress.hpp" - -namespace ZeroTier { - -/** - * Loads a GeoIP CSV into memory for fast lookup, reloading as needed - * - * This was designed around the CSV from https://db-ip.com but can be used - * with any similar GeoIP CSV database that is presented in the form of an - * IP range and lat/long coordinates. - * - * It loads the whole database into memory, which can be kind of large. If - * the CSV file changes, the changes are loaded automatically. - */ -class ClusterGeoIpService : NonCopyable -{ -public: - ClusterGeoIpService(); - ~ClusterGeoIpService(); - - /** - * Load or reload CSV file - * - * CSV column indexes start at zero. CSVs can be quoted with single or - * double quotes. Whitespace before or after commas is ignored. Backslash - * may be used for escaping whitespace as well. - * - * @param pathToCsv Path to (uncompressed) CSV file - * @param ipStartColumn Column with IP range start - * @param ipEndColumn Column with IP range end (inclusive) - * @param latitudeColumn Column with latitude - * @param longitudeColumn Column with longitude - * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) - */ - inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) - { - Mutex::Lock _l(_lock); - return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); - } - - /** - * Attempt to locate an IP - * - * This returns true if x, y, and z are set. If the return value is false - * the values of x, y, and z are undefined. - * - * @param ip IPv4 or IPv6 address - * @param x Reference to variable to receive X - * @param y Reference to variable to receive Y - * @param z Reference to variable to receive Z - * @return True if coordinates were set - */ - bool locate(const InetAddress &ip,int &x,int &y,int &z); - - /** - * @return True if IP database/service is available for queries (otherwise locate() will always be false) - */ - inline bool available() const - { - Mutex::Lock _l(_lock); - return ((_v4db.size() + _v6db.size()) > 0); - } - -private: - struct _V4E - { - uint32_t start; - uint32_t end; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V4E &e) const { return (start < e.start); } - }; - - struct _V6E - { - uint8_t start[16]; - uint8_t end[16]; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } - }; - - static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - - std::string _pathToCsv; - int _ipStartColumn; - int _ipEndColumn; - int _latitudeColumn; - int _longitudeColumn; - - uint64_t _lastFileCheckTime; - uint64_t _csvModificationTime; - int64_t _csvFileSize; - - std::vector<_V4E> _v4db; - std::vector<_V6E> _v6db; - - Mutex _lock; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif -- cgit v1.2.3 From dff8c02cfee9eaafae0974f3b070ff849a94c4ac Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 12:33:00 -0700 Subject: Pull out and deprecate old cluster code. New cluster code will not be merged yet. --- node/IncomingPacket.cpp | 12 ++++++------ node/Peer.cpp | 18 ++++++++++++++++-- node/Peer.hpp | 17 ++++++++++++++++- service/OneService.cpp | 20 ++++++++++++++++++++ 4 files changed, 58 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index f0be96f9..ac8514c6 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1199,9 +1199,9 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { - //if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) - // peer->setClusterPreferred(a); - if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { + peer->redirect(tPtr,_path->localSocket(),a,now); + } else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { @@ -1216,9 +1216,9 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { - //if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) - // peer->setClusterPreferred(a); - if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { + peer->redirect(tPtr,_path->localSocket(),a,now); + } else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } else { diff --git a/node/Peer.cpp b/node/Peer.cpp index fb9a72b1..e16540b3 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -170,11 +170,11 @@ void Peer::received( Mutex::Lock _l(_paths_m); _PeerPath *potentialNewPeerPath = (_PeerPath *)0; if (path->address().ss_family == AF_INET) { - if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) { + if ( ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) && ( (now - _v4Path.sticky) > ZT_PEER_PATH_EXPIRATION ) ) { potentialNewPeerPath = &_v4Path; } } else if (path->address().ss_family == AF_INET6) { - if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) { + if ( ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) && ( (now - _v6Path.sticky) > ZT_PEER_PATH_EXPIRATION ) ) { potentialNewPeerPath = &_v6Path; } } @@ -422,4 +422,18 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } +void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now) +{ + Mutex::Lock _l(_paths_m); + SharedPtr p(RR->topology->getPath(localSocket,remoteAddress)); + attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,p->nextOutgoingCounter()); + if (remoteAddress.ss_family == AF_INET) { + _v4Path.p = p; + _v4Path.sticky = now; + } else if (remoteAddress.ss_family == AF_INET6) { + _v6Path.p = p; + _v6Path.sticky = now; + } +} + } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index ad2d0ddc..b24318ec 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -195,6 +195,20 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); + /** + * Specify remote path for this peer and forget others + * + * This overrides normal path learning and tells this peer to be found + * at this address, at least within the address's family. Other address + * families are not modified. + * + * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param localSocket Local socket as supplied by external code + * @param remoteAddress Remote address + * @param now Current time + */ + void redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now); + /** * Reset paths within a given IP scope and address family * @@ -426,8 +440,9 @@ public: private: struct _PeerPath { - _PeerPath() : lr(0),p() {} + _PeerPath() : lr(0),sticky(0),p() {} uint64_t lr; // time of last valid ZeroTier packet + uint64_t sticky; // time last set as sticky SharedPtr p; }; diff --git a/service/OneService.cpp b/service/OneService.cpp index 6497ae20..6c2c9a8b 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -394,6 +394,8 @@ public: Phy _phy; Node *_node; SoftwareUpdater *_updater; + PhySocket *_localControlSocket4; + PhySocket *_localControlSocket6; bool _updateAutoApply; unsigned int _primaryPort; volatile unsigned int _udpPortPickerCounter; @@ -488,6 +490,8 @@ public: ,_phy(this,false,true) ,_node((Node *)0) ,_updater((SoftwareUpdater *)0) + ,_localControlSocket4((PhySocket *)0) + ,_localControlSocket6((PhySocket *)0) ,_updateAutoApply(false) ,_primaryPort(port) ,_udpPortPickerCounter(0) @@ -513,6 +517,8 @@ public: virtual ~OneServiceImpl() { _binder.closeAll(_phy); + _phy.close(_localControlSocket4); + _phy.close(_localControlSocket6); #ifdef ZT_USE_MINIUPNPC delete _portMapper; #endif @@ -652,6 +658,20 @@ public: return _termReason; } + // Bind local control socket + { + struct sockaddr_in lo4; + memset(&lo4,0,sizeof(lo4)); + lo4.sin_family = AF_INET; + lo4.sin_port = Utils::hton((uint16_t)_ports[0]); + _localControlSocket4 = _phy.tcpListen((const struct sockaddr *)&lo4); + struct sockaddr_in6 lo6; + memset(&lo6,0,sizeof(lo6)); + lo6.sin6_family = AF_INET6; + lo6.sin6_port = lo4.sin_port; + _localControlSocket6 = _phy.tcpListen((const struct sockaddr *)&lo6); + } + // Save primary port to a file so CLIs and GUIs can learn it easily char portstr[64]; Utils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); -- cgit v1.2.3 From d2415dee00914ab3fd7016758f4184d46bb407a5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 16:11:11 -0700 Subject: Cleanup. --- controller/EmbeddedNetworkController.cpp | 104 +++++++++-------- controller/JSONDB.cpp | 14 +-- node/Address.hpp | 15 +-- node/CertificateOfMembership.cpp | 7 +- node/Dictionary.hpp | 6 +- node/Identity.cpp | 26 +++-- node/Identity.hpp | 18 +-- node/InetAddress.cpp | 195 ++++++++++++++----------------- node/InetAddress.hpp | 94 ++++++--------- node/MAC.hpp | 83 +++---------- node/MulticastGroup.hpp | 12 -- node/Network.cpp | 14 ++- node/NetworkConfig.cpp | 12 +- node/Node.cpp | 51 ++------ node/Node.hpp | 4 - node/RuntimeEnvironment.hpp | 10 +- node/Topology.cpp | 8 +- node/Utils.cpp | 114 ++++-------------- node/Utils.hpp | 182 +++++++++++++++++++++-------- one.cpp | 55 +++++---- osdep/BSDEthernetTap.cpp | 19 +-- osdep/Http.cpp | 4 +- osdep/LinuxEthernetTap.cpp | 33 +++--- osdep/ManagedRoute.cpp | 11 +- osdep/OSUtils.cpp | 28 ++++- osdep/OSUtils.hpp | 15 ++- osdep/OSXEthernetTap.cpp | 20 ++-- osdep/PortMapper.cpp | 4 +- osdep/WindowsEthernetTap.cpp | 10 +- selftest.cpp | 31 +++-- service/OneService.cpp | 94 +++++++-------- service/SoftwareUpdater.cpp | 5 +- 32 files changed, 620 insertions(+), 678 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 85c759e7..b57a37e8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -76,19 +76,19 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_ACTION_TEE: r["type"] = "ACTION_TEE"; - r["address"] = Address(rule.v.fwd.address).toString(); + r["address"] = Address(rule.v.fwd.address).toString(tmp); r["flags"] = (unsigned int)rule.v.fwd.flags; r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_WATCH: r["type"] = "ACTION_WATCH"; - r["address"] = Address(rule.v.fwd.address).toString(); + r["address"] = Address(rule.v.fwd.address).toString(tmp); r["flags"] = (unsigned int)rule.v.fwd.flags; r["length"] = (unsigned int)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; - r["address"] = Address(rule.v.fwd.address).toString(); + r["address"] = Address(rule.v.fwd.address).toString(tmp); r["flags"] = (unsigned int)rule.v.fwd.flags; break; case ZT_NETWORK_RULE_ACTION_BREAK: @@ -102,11 +102,11 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) switch(rt) { case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); + r["zt"] = Address(rule.v.zt).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); + r["zt"] = Address(rule.v.zt).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: r["type"] = "MATCH_VLAN_ID"; @@ -122,29 +122,29 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: r["type"] = "MATCH_IPV4_SOURCE"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: r["type"] = "MATCH_IPV4_DEST"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: r["type"] = "MATCH_IPV6_SOURCE"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: r["type"] = "MATCH_IPV6_DEST"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(tmp); break; case ZT_NETWORK_RULE_MATCH_IP_TOS: r["type"] = "MATCH_IP_TOS"; @@ -179,7 +179,7 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: r["type"] = "MATCH_CHARACTERISTICS"; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics); r["mask"] = tmp; break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: @@ -312,28 +312,28 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "MATCH_IPV4_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0").c_str()); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV4_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"0.0.0.0").c_str()); rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; return true; } else if (t == "MATCH_IPV6_SOURCE") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(OSUtils::jsonString(r["ip"],"::0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"::0").c_str()); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; return true; } else if (t == "MATCH_IPV6_DEST") { rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(OSUtils::jsonString(r["ip"],"::0")); + InetAddress ip(OSUtils::jsonString(r["ip"],"::0").c_str()); memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; @@ -514,7 +514,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { char tmp[128]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s%.10llx\":%llu",(responseBody.length() > 1) ? ",\"" : "\"",(unsigned long long)nodeId,(unsigned long long)OSUtils::jsonInt(member["revision"],0)); responseBody.append(tmp); } }); @@ -548,7 +548,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { if (responseBody.length() > 1) responseBody.push_back(','); - Utils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); responseBody.append(tmp); } responseBody.push_back(']'); @@ -562,7 +562,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( // Controller status char tmp[4096]; - Utils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); responseBody = tmp; responseContentType = "application/json"; return 200; @@ -603,14 +603,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if ((path.size() >= 2)&&(path[1].length() == 16)) { uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); char nwids[24]; - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { uint64_t address = Utils::hexStrToU64(path[3].c_str()); char addrs[24]; - Utils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + OSUtils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member; _db.getNetworkMember(nwid,address,member); @@ -655,9 +655,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json mipa(json::array()); for(unsigned long i=0;i()); + InetAddress t(target.get().c_str()); InetAddress v; - if (via.is_string()) v.fromString(via.get()); + if (via.is_string()) v.fromString(via.get().c_str()); if ( ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) && (t.netmaskBitsValid()) ) { json tmp; - tmp["target"] = t.toString(); + char tmp2[64]; + tmp["target"] = t.toString(tmp2); if (v.ss_family == t.ss_family) - tmp["via"] = v.toIpString(); + tmp["via"] = v.toIpString(tmp2); else tmp["via"] = json(); nrts.push_back(tmp); } @@ -840,12 +842,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( for(unsigned long i=0;i_memberStatus.find(_MemberStatusKey(networkId,nodeId)); if (ms != _memberStatus.end()) lrt = ms->second.lastRequestTime; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", (first) ? "" : ",", (unsigned long long)networkId, (unsigned long long)nodeId, @@ -1093,7 +1096,7 @@ void EmbeddedNetworkController::threadMain() }); } char tmp2[256]; - Utils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + OSUtils::ztsnprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); pong.append(tmp2); _db.writeRaw("pong",pong); } @@ -1126,7 +1129,7 @@ void EmbeddedNetworkController::_request( ms.lastRequestTime = now; } - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; @@ -1152,13 +1155,15 @@ void EmbeddedNetworkController::_request( } } else { // If we do not yet know this member's identity, learn it. - member["identity"] = identity.toString(false); + char idtmp[1024]; + member["identity"] = identity.toString(false,idtmp); } } // These are always the same, but make sure they are set { - const std::string addrs(identity.address().toString()); + char tmpid[128]; + const std::string addrs(identity.address().toString(tmpid)); member["id"] = addrs; member["address"] = addrs; member["nwid"] = nwids; @@ -1264,8 +1269,9 @@ void EmbeddedNetworkController::_request( if (fromAddr) ms.physicalAddr = fromAddr; + char tmpip[64]; if (ms.physicalAddr) - member["physicalAddr"] = ms.physicalAddr.toString(); + member["physicalAddr"] = ms.physicalAddr.toString(tmpip); } } } else { @@ -1427,9 +1433,9 @@ void EmbeddedNetworkController::_request( json &target = route["target"]; json &via = route["via"]; if (target.is_string()) { - const InetAddress t(target.get()); + const InetAddress t(target.get().c_str()); InetAddress v; - if (via.is_string()) v.fromString(via.get()); + if (via.is_string()) v.fromString(via.get().c_str()); if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { ZT_VirtualNetworkRoute *r = &(nc->routes[nc->routeCount]); *(reinterpret_cast(&(r->target))) = t; @@ -1462,7 +1468,7 @@ void EmbeddedNetworkController::_request( if (!ipAssignments[i].is_string()) continue; std::string ips = ipAssignments[i]; - InetAddress ip(ips); + InetAddress ip(ips.c_str()); // IP assignments are only pushed if there is a corresponding local route. We also now get the netmask bits from // this route, ignoring the netmask bits field of the assigned IP itself. Using that was worthless and a source @@ -1492,8 +1498,8 @@ void EmbeddedNetworkController::_request( for(unsigned long p=0;((p 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip6)) ) { - ipAssignments.push_back(ip6.toIpString()); + char tmpip[64]; + ipAssignments.push_back(ip6.toIpString(tmpip)); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) @@ -1552,8 +1559,8 @@ void EmbeddedNetworkController::_request( for(unsigned long p=0;((p(&ipRangeStartIA)->sin_addr.s_addr)); uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); @@ -1586,7 +1593,8 @@ void EmbeddedNetworkController::_request( // If it's routed, then try to claim and assign it and if successful end loop const InetAddress ip4(Utils::hton(ip),0); if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip4)) ) { - ipAssignments.push_back(ip4.toIpString()); + char tmpip[64]; + ipAssignments.push_back(ip4.toIpString(tmpip)); member["ipAssignments"] = ipAssignments; if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { struct sockaddr_in *const v4ip = reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index acf23700..97a217a1 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -39,7 +39,7 @@ JSONDB::JSONDB(const std::string &basePath) : std::size_t hnsep = hn.find_last_of(':'); if (hnsep != std::string::npos) hn[hnsep] = '/'; - _httpAddr.fromString(hn); + _httpAddr.fromString(hn.c_str()); if (hnend != std::string::npos) _basePath = _basePath.substr(7 + hnend); if (_basePath.length() == 0) @@ -94,7 +94,7 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) std::string body; std::map reqHeaders; char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); reqHeaders["Content-Length"] = tmp; reqHeaders["Content-Type"] = "application/json"; const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); @@ -164,7 +164,7 @@ bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlo void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) { char n[64]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); writeRaw(n,OSUtils::jsonDump(networkConfig)); { Mutex::Lock _l(_networks_m); @@ -176,7 +176,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) { char n[256]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); writeRaw(n,OSUtils::jsonDump(memberConfig)); { Mutex::Lock _l(_networks_m); @@ -202,7 +202,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) } char n[256]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); if (_httpAddr) { // Deletion is currently done by Central in harnessed mode @@ -229,7 +229,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) { char n[256]; - Utils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); if (_httpAddr) { // Deletion is currently done by the caller in Central harnessed mode @@ -314,7 +314,7 @@ void JSONDB::threadMain() const nlohmann::json &mips = member["ipAssignments"]; if (mips.is_array()) { for(unsigned long i=0;iadd(key,tmp,-1); + return this->add(key,Utils::hex(value,tmp),-1); } /** @@ -401,8 +400,7 @@ public: inline bool add(const char *key,const Address &a) { char tmp[32]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",(unsigned long long)a.toInt()); - return this->add(key,tmp,-1); + return this->add(key,Utils::hex(a.toInt(),tmp),-1); } /** diff --git a/node/Identity.cpp b/node/Identity.cpp index ba77aa47..3b00b4c0 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -136,19 +136,23 @@ bool Identity::locallyValidate() const (digest[63] == addrb[4])); } -std::string Identity::toString(bool includePrivate) const +char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const { - std::string r; - - r.append(_address.toString()); - r.append(":0:"); // 0 == ZT_OBJECT_TYPE_IDENTITY - r.append(Utils::hex(_publicKey.data,(unsigned int)_publicKey.size())); + char *p = buf; + Utils::hex10(_address.toInt(),p); + p += 10; + *(p++) = ':'; + *(p++) = '0'; + *(p++) = ':'; + Utils::hex(_publicKey.data,ZT_C25519_PUBLIC_KEY_LEN,p); + p += ZT_C25519_PUBLIC_KEY_LEN * 2; if ((_privateKey)&&(includePrivate)) { - r.push_back(':'); - r.append(Utils::hex(_privateKey->data,(unsigned int)_privateKey->size())); + *(p++) = ':'; + Utils::hex(_privateKey->data,ZT_C25519_PRIVATE_KEY_LEN,p); + p += ZT_C25519_PRIVATE_KEY_LEN * 2; } - - return r; + *(p++) = (char)0; + return buf; } bool Identity::fromString(const char *str) @@ -157,7 +161,7 @@ bool Identity::fromString(const char *str) return false; char *saveptr = (char *)0; - char tmp[1024]; + char tmp[ZT_IDENTITY_STRING_BUFFER_LENGTH]; if (!Utils::scopy(tmp,sizeof(tmp),str)) return false; diff --git a/node/Identity.hpp b/node/Identity.hpp index 79e17f4d..5804b9f8 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -29,7 +29,6 @@ #include #include -#include #include "Constants.hpp" #include "Array.hpp" @@ -39,6 +38,8 @@ #include "Buffer.hpp" #include "SHA512.hpp" +#define ZT_IDENTITY_STRING_BUFFER_LENGTH 384 + namespace ZeroTier { /** @@ -66,16 +67,7 @@ public: { } - Identity(const char *str) - throw(std::invalid_argument) : - _privateKey((C25519::Private *)0) - { - if (!fromString(str)) - throw std::invalid_argument(std::string("invalid string-serialized identity: ") + str); - } - - Identity(const std::string &str) - throw(std::invalid_argument) : + Identity(const char *str) : _privateKey((C25519::Private *)0) { if (!fromString(str)) @@ -277,9 +269,10 @@ public: * Serialize to a more human-friendly string * * @param includePrivate If true, include private key (if it exists) + * @param buf Buffer to store string * @return ASCII string representation of identity */ - std::string toString(bool includePrivate) const; + char *toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const; /** * Deserialize a human-friendly string @@ -291,7 +284,6 @@ public: * @return True if deserialization appears successful */ bool fromString(const char *str); - inline bool fromString(const std::string &str) { return fromString(str.c_str()); } /** * @return C25519 public key diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index 17d7c72e..f7585bdb 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -5,7 +5,7 @@ * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * (at your oion) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -40,7 +40,6 @@ const InetAddress InetAddress::LO4((const void *)("\x7f\x00\x00\x01"),4,0); const InetAddress InetAddress::LO6((const void *)("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01"),16,0); InetAddress::IpScope InetAddress::ipScope() const - throw() { switch(ss_family) { @@ -111,27 +110,7 @@ InetAddress::IpScope InetAddress::ipScope() const return IP_SCOPE_NONE; } -void InetAddress::set(const std::string &ip,unsigned int port) - throw() -{ - memset(this,0,sizeof(InetAddress)); - if (ip.find(':') != std::string::npos) { - struct sockaddr_in6 *sin6 = reinterpret_cast(this); - ss_family = AF_INET6; - sin6->sin6_port = Utils::hton((uint16_t)port); - if (inet_pton(AF_INET6,ip.c_str(),(void *)&(sin6->sin6_addr.s6_addr)) <= 0) - memset(this,0,sizeof(InetAddress)); - } else if (ip.find('.') != std::string::npos) { - struct sockaddr_in *sin = reinterpret_cast(this); - ss_family = AF_INET; - sin->sin_port = Utils::hton((uint16_t)port); - if (inet_pton(AF_INET,ip.c_str(),(void *)&(sin->sin_addr.s_addr)) <= 0) - memset(this,0,sizeof(InetAddress)); - } -} - void InetAddress::set(const void *ipBytes,unsigned int ipLen,unsigned int port) - throw() { memset(this,0,sizeof(InetAddress)); if (ipLen == 4) { @@ -147,90 +126,98 @@ void InetAddress::set(const void *ipBytes,unsigned int ipLen,unsigned int port) } } -std::string InetAddress::toString() const +char *InetAddress::toString(char buf[64]) const { - char buf[128]; - switch(ss_family) { - case AF_INET: - Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d/%d", - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[3], - (int)Utils::ntoh((uint16_t)(reinterpret_cast(this)->sin_port)) - ); - return std::string(buf); - case AF_INET6: - Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x/%d", - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[3]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[4]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[5]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[6]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[7]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[8]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[9]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[10]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[11]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[12]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[13]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[14]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[15]), - (int)Utils::ntoh((uint16_t)(reinterpret_cast(this)->sin6_port)) - ); - return std::string(buf); + char *p = toIpString(buf); + if (*p) { + while (*p) ++p; + *(p++) = '/'; + Utils::decimal(port(),p); } - return std::string(); + return buf; } -std::string InetAddress::toIpString() const +char *InetAddress::toIpString(char buf[64]) const { - char buf[128]; switch(ss_family) { - case AF_INET: - Utils::ztsnprintf(buf,sizeof(buf),"%d.%d.%d.%d", - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[0], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[1], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[2], - (int)(reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)))[3] - ); - return std::string(buf); - case AF_INET6: - Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[0]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[1]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[2]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[3]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[4]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[5]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[6]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[7]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[8]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[9]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[10]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[11]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[12]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[13]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[14]), - (int)(reinterpret_cast(this)->sin6_addr.s6_addr[15]) - ); - return std::string(buf); + case AF_INET: { + const uint8_t *a = reinterpret_cast(&(reinterpret_cast(this)->sin_addr.s_addr)); + char *p = buf; + for(int i=0;;++i) { + Utils::decimal((unsigned long)a[i],p); + if (i != 3) { + while (*p) ++p; + *(p++) = '.'; + } else break; + } + } break; + + case AF_INET6: { + uint16_t a[8]; + memcpy(a,reinterpret_cast(this)->sin6_addr.s6_addr,16); + char *p = buf; + for(int i=0;i<8;++i) { + Utils::hex(Utils::ntoh(a[i]),p); + p[4] = (i == 7) ? (char)0 : ':'; + p += 5; + } + } break; + + default: + buf[0] = (char)0; + break; } - return std::string(); + return buf; } -void InetAddress::fromString(const std::string &ipSlashPort) +bool InetAddress::fromString(const char *ipSlashPort) { - const std::size_t slashAt = ipSlashPort.find('/'); - if (slashAt == std::string::npos) { - set(ipSlashPort,0); + char buf[64]; + + memset(this,0,sizeof(InetAddress)); + + if (!*ipSlashPort) + return true; + if (!Utils::scopy(buf,sizeof(buf),ipSlashPort)) + return false; + + char *portAt = buf; + while ((*portAt)&&(*portAt != '/')) + ++portAt; + unsigned int port = 0; + if (*portAt) { + *(portAt++) = (char)0; + port = Utils::strToUInt(portAt) & 0xffff; + } + + if (strchr(buf,':')) { + uint16_t a[8]; + unsigned int b = 0; + char *saveptr = (char *)0; + for(char *s=Utils::stok(buf,":",&saveptr);((s)&&(b<8));s=Utils::stok((char *)0,":",&saveptr)) + a[b++] = Utils::hton((uint16_t)(Utils::hexStrToUInt(s) & 0xffff)); + + struct sockaddr_in6 *const in6 = reinterpret_cast(this); + in6->sin6_family = AF_INET6; + memcpy(in6->sin6_addr.s6_addr,a,16); + in6->sin6_port = Utils::hton((uint16_t)port); + + return true; + } else if (strchr(buf,'.')) { + uint8_t a[4]; + unsigned int b = 0; + char *saveptr = (char *)0; + for(char *s=Utils::stok(buf,".",&saveptr);((s)&&(b<4));s=Utils::stok((char *)0,".",&saveptr)) + a[b++] = (uint8_t)(Utils::strToUInt(s) & 0xff); + + struct sockaddr_in *const in = reinterpret_cast(this); + in->sin_family = AF_INET; + memcpy(&(in->sin_addr.s_addr),a,4); + in->sin_port = Utils::hton((uint16_t)port); + + return true; } else { - long p = strtol(ipSlashPort.substr(slashAt+1).c_str(),(char **)0,10); - if ((p > 0)&&(p <= 0xffff)) - set(ipSlashPort.substr(0,slashAt),(unsigned int)p); - else set(ipSlashPort.substr(0,slashAt),0); + return false; } } @@ -244,14 +231,13 @@ InetAddress InetAddress::netmask() const case AF_INET6: { uint64_t nm[2]; const unsigned int bits = netmaskBits(); - if(bits) { - nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); - nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); - } - else { - nm[0] = 0; - nm[1] = 0; - } + if(bits) { + nm[0] = Utils::hton((uint64_t)((bits >= 64) ? 0xffffffffffffffffULL : (0xffffffffffffffffULL << (64 - bits)))); + nm[1] = Utils::hton((uint64_t)((bits <= 64) ? 0ULL : (0xffffffffffffffffULL << (128 - bits)))); + } else { + nm[0] = 0; + nm[1] = 0; + } memcpy(reinterpret_cast(&r)->sin6_addr.s6_addr,nm,16); } break; } @@ -338,7 +324,6 @@ bool InetAddress::containsAddress(const InetAddress &addr) const } bool InetAddress::isNetwork() const - throw() { switch(ss_family) { case AF_INET: { @@ -371,7 +356,6 @@ bool InetAddress::isNetwork() const } bool InetAddress::operator==(const InetAddress &a) const - throw() { if (ss_family == a.ss_family) { switch(ss_family) { @@ -395,7 +379,6 @@ bool InetAddress::operator==(const InetAddress &a) const } bool InetAddress::operator<(const InetAddress &a) const - throw() { if (ss_family < a.ss_family) return true; diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 4cb9a4dc..dd055084 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -31,8 +31,6 @@ #include #include -#include - #include "Constants.hpp" #include "../include/ZeroTierOne.h" #include "Utils.hpp" @@ -85,25 +83,22 @@ struct InetAddress : public sockaddr_storage IP_SCOPE_PRIVATE = 7 // 10.x.x.x, 192.168.x.x, etc. }; - InetAddress() throw() { memset(this,0,sizeof(InetAddress)); } - InetAddress(const InetAddress &a) throw() { memcpy(this,&a,sizeof(InetAddress)); } - InetAddress(const InetAddress *a) throw() { memcpy(this,a,sizeof(InetAddress)); } - InetAddress(const struct sockaddr_storage &ss) throw() { *this = ss; } - InetAddress(const struct sockaddr_storage *ss) throw() { *this = ss; } - InetAddress(const struct sockaddr &sa) throw() { *this = sa; } - InetAddress(const struct sockaddr *sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in &sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in *sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in6 &sa) throw() { *this = sa; } - InetAddress(const struct sockaddr_in6 *sa) throw() { *this = sa; } - InetAddress(const void *ipBytes,unsigned int ipLen,unsigned int port) throw() { this->set(ipBytes,ipLen,port); } - InetAddress(const uint32_t ipv4,unsigned int port) throw() { this->set(&ipv4,4,port); } - InetAddress(const std::string &ip,unsigned int port) throw() { this->set(ip,port); } - InetAddress(const std::string &ipSlashPort) throw() { this->fromString(ipSlashPort); } - InetAddress(const char *ipSlashPort) throw() { this->fromString(std::string(ipSlashPort)); } + InetAddress() { memset(this,0,sizeof(InetAddress)); } + InetAddress(const InetAddress &a) { memcpy(this,&a,sizeof(InetAddress)); } + InetAddress(const InetAddress *a) { memcpy(this,a,sizeof(InetAddress)); } + InetAddress(const struct sockaddr_storage &ss) { *this = ss; } + InetAddress(const struct sockaddr_storage *ss) { *this = ss; } + InetAddress(const struct sockaddr &sa) { *this = sa; } + InetAddress(const struct sockaddr *sa) { *this = sa; } + InetAddress(const struct sockaddr_in &sa) { *this = sa; } + InetAddress(const struct sockaddr_in *sa) { *this = sa; } + InetAddress(const struct sockaddr_in6 &sa) { *this = sa; } + InetAddress(const struct sockaddr_in6 *sa) { *this = sa; } + InetAddress(const void *ipBytes,unsigned int ipLen,unsigned int port) { this->set(ipBytes,ipLen,port); } + InetAddress(const uint32_t ipv4,unsigned int port) { this->set(&ipv4,4,port); } + InetAddress(const char *ipSlashPort) { this->fromString(ipSlashPort); } inline InetAddress &operator=(const InetAddress &a) - throw() { if (&a != this) memcpy(this,&a,sizeof(InetAddress)); @@ -111,7 +106,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const InetAddress *a) - throw() { if (a != this) memcpy(this,a,sizeof(InetAddress)); @@ -119,7 +113,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_storage &ss) - throw() { if (reinterpret_cast(&ss) != this) memcpy(this,&ss,sizeof(InetAddress)); @@ -127,7 +120,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_storage *ss) - throw() { if (reinterpret_cast(ss) != this) memcpy(this,ss,sizeof(InetAddress)); @@ -135,7 +127,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in &sa) - throw() { if (reinterpret_cast(&sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -145,7 +136,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in *sa) - throw() { if (reinterpret_cast(sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -155,7 +145,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in6 &sa) - throw() { if (reinterpret_cast(&sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -165,7 +154,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr_in6 *sa) - throw() { if (reinterpret_cast(sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -175,7 +163,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr &sa) - throw() { if (reinterpret_cast(&sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -192,7 +179,6 @@ struct InetAddress : public sockaddr_storage } inline InetAddress &operator=(const struct sockaddr *sa) - throw() { if (reinterpret_cast(sa) != this) { memset(this,0,sizeof(InetAddress)); @@ -211,17 +197,7 @@ struct InetAddress : public sockaddr_storage /** * @return IP scope classification (e.g. loopback, link-local, private, global) */ - IpScope ipScope() const - throw(); - - /** - * Set from a string-format IP and a port - * - * @param ip IP address in V4 or V6 ASCII notation - * @param port Port or 0 for none - */ - void set(const std::string &ip,unsigned int port) - throw(); + IpScope ipScope() const; /** * Set from a raw IP and port number @@ -230,8 +206,7 @@ struct InetAddress : public sockaddr_storage * @param ipLen Length of IP address: 4 or 16 * @param port Port number or 0 for none */ - void set(const void *ipBytes,unsigned int ipLen,unsigned int port) - throw(); + void set(const void *ipBytes,unsigned int ipLen,unsigned int port); /** * Set the port component @@ -272,23 +247,23 @@ struct InetAddress : public sockaddr_storage /** * @return ASCII IP/port format representation */ - std::string toString() const; + char *toString(char buf[64]) const; /** * @return IP portion only, in ASCII string format */ - std::string toIpString() const; + char *toIpString(char buf[64]) const; /** - * @param ipSlashPort ASCII IP/port format notation + * @param ipSlashPort IP/port (port is optional, will be 0 if not included) + * @return True if address appeared to be valid */ - void fromString(const std::string &ipSlashPort); + bool fromString(const char *ipSlashPort); /** * @return Port or 0 if no port component defined */ inline unsigned int port() const - throw() { switch(ss_family) { case AF_INET: return Utils::ntoh((uint16_t)(reinterpret_cast(this)->sin_port)); @@ -306,7 +281,7 @@ struct InetAddress : public sockaddr_storage * * @return Netmask bits */ - inline unsigned int netmaskBits() const throw() { return port(); } + inline unsigned int netmaskBits() const { return port(); } /** * @return True if netmask bits is valid for the address type @@ -329,7 +304,7 @@ struct InetAddress : public sockaddr_storage * * @return Gateway metric */ - inline unsigned int metric() const throw() { return port(); } + inline unsigned int metric() const { return port(); } /** * Construct a full netmask as an InetAddress @@ -376,12 +351,12 @@ struct InetAddress : public sockaddr_storage /** * @return True if this is an IPv4 address */ - inline bool isV4() const throw() { return (ss_family == AF_INET); } + inline bool isV4() const { return (ss_family == AF_INET); } /** * @return True if this is an IPv6 address */ - inline bool isV6() const throw() { return (ss_family == AF_INET6); } + inline bool isV6() const { return (ss_family == AF_INET6); } /** * @return pointer to raw address bytes or NULL if not available @@ -454,7 +429,7 @@ struct InetAddress : public sockaddr_storage /** * Set to null/zero */ - inline void zero() throw() { memset(this,0,sizeof(InetAddress)); } + inline void zero() { memset(this,0,sizeof(InetAddress)); } /** * Check whether this is a network/route rather than an IP assignment @@ -464,8 +439,7 @@ struct InetAddress : public sockaddr_storage * * @return True if everything after netmask bits is zero */ - bool isNetwork() const - throw(); + bool isNetwork() const; /** * @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP @@ -494,7 +468,7 @@ struct InetAddress : public sockaddr_storage /** * @return True if address family is non-zero */ - inline operator bool() const throw() { return (ss_family != 0); } + inline operator bool() const { return (ss_family != 0); } template inline void serialize(Buffer &b) const @@ -552,12 +526,12 @@ struct InetAddress : public sockaddr_storage return (p - startAt); } - bool operator==(const InetAddress &a) const throw(); - bool operator<(const InetAddress &a) const throw(); - inline bool operator!=(const InetAddress &a) const throw() { return !(*this == a); } - inline bool operator>(const InetAddress &a) const throw() { return (a < *this); } - inline bool operator<=(const InetAddress &a) const throw() { return !(a < *this); } - inline bool operator>=(const InetAddress &a) const throw() { return !(*this < a); } + bool operator==(const InetAddress &a) const; + bool operator<(const InetAddress &a) const; + inline bool operator!=(const InetAddress &a) const { return !(*this == a); } + inline bool operator>(const InetAddress &a) const { return (a < *this); } + inline bool operator<=(const InetAddress &a) const { return !(a < *this); } + inline bool operator>=(const InetAddress &a) const { return !(*this < a); } /** * @param mac MAC address seed diff --git a/node/MAC.hpp b/node/MAC.hpp index db50aeb1..52388d59 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -44,30 +44,24 @@ namespace ZeroTier { class MAC { public: - MAC() throw() : _m(0ULL) {} - MAC(const MAC &m) throw() : _m(m._m) {} + MAC() : _m(0ULL) {} + MAC(const MAC &m) : _m(m._m) {} - MAC(const unsigned char a,const unsigned char b,const unsigned char c,const unsigned char d,const unsigned char e,const unsigned char f) throw() : + MAC(const unsigned char a,const unsigned char b,const unsigned char c,const unsigned char d,const unsigned char e,const unsigned char f) : _m( ((((uint64_t)a) & 0xffULL) << 40) | ((((uint64_t)b) & 0xffULL) << 32) | ((((uint64_t)c) & 0xffULL) << 24) | ((((uint64_t)d) & 0xffULL) << 16) | ((((uint64_t)e) & 0xffULL) << 8) | (((uint64_t)f) & 0xffULL) ) {} - - MAC(const char *s) throw() { fromString(s); } - MAC(const std::string &s) throw() { fromString(s.c_str()); } - - MAC(const void *bits,unsigned int len) throw() { setTo(bits,len); } - - MAC(const Address &ztaddr,uint64_t nwid) throw() { fromAddress(ztaddr,nwid); } - - MAC(const uint64_t m) throw() : _m(m & 0xffffffffffffULL) {} + MAC(const void *bits,unsigned int len) { setTo(bits,len); } + MAC(const Address &ztaddr,uint64_t nwid) { fromAddress(ztaddr,nwid); } + MAC(const uint64_t m) : _m(m & 0xffffffffffffULL) {} /** * @return MAC in 64-bit integer */ - inline uint64_t toInt() const throw() { return _m; } + inline uint64_t toInt() const { return _m; } /** * Set MAC to zero @@ -77,14 +71,13 @@ public: /** * @return True if MAC is non-zero */ - inline operator bool() const throw() { return (_m != 0ULL); } + inline operator bool() const { return (_m != 0ULL); } /** * @param bits Raw MAC in big-endian byte order * @param len Length, must be >= 6 or result is zero */ inline void setTo(const void *bits,unsigned int len) - throw() { if (len < 6) { _m = 0ULL; @@ -104,7 +97,6 @@ public: * @param len Length of buffer, must be >= 6 or nothing is copied */ inline void copyTo(void *buf,unsigned int len) const - throw() { if (len < 6) return; @@ -124,7 +116,6 @@ public: */ template inline void appendTo(Buffer &b) const - throw(std::out_of_range) { unsigned char *p = (unsigned char *)b.appendField(6); *(p++) = (unsigned char)((_m >> 40) & 0xff); @@ -138,48 +129,17 @@ public: /** * @return True if this is broadcast (all 0xff) */ - inline bool isBroadcast() const throw() { return (_m == 0xffffffffffffULL); } + inline bool isBroadcast() const { return (_m == 0xffffffffffffULL); } /** * @return True if this is a multicast MAC */ - inline bool isMulticast() const throw() { return ((_m & 0x010000000000ULL) != 0ULL); } + inline bool isMulticast() const { return ((_m & 0x010000000000ULL) != 0ULL); } /** * @param True if this is a locally-administered MAC */ - inline bool isLocallyAdministered() const throw() { return ((_m & 0x020000000000ULL) != 0ULL); } - - /** - * @param s Hex MAC, with or without : delimiters - */ - inline void fromString(const char *s) - { - char tmp[8]; - for(int i=0;i<6;++i) - tmp[i] = (char)0; - Utils::unhex(s,tmp,6); - setTo(tmp,6); - } - - /** - * @return MAC address in standard :-delimited hex format - */ - inline std::string toString() const - { - char tmp[24]; - toString(tmp,sizeof(tmp)); - return std::string(tmp); - } - - /** - * @param buf Buffer to contain human-readable MAC - * @param len Length of buffer - */ - inline void toString(char *buf,unsigned int len) const - { - Utils::ztsnprintf(buf,len,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)(*this)[0],(int)(*this)[1],(int)(*this)[2],(int)(*this)[3],(int)(*this)[4],(int)(*this)[5]); - } + inline bool isLocallyAdministered() const { return ((_m & 0x020000000000ULL) != 0ULL); } /** * Set this MAC to a MAC derived from an address and a network ID @@ -188,7 +148,6 @@ public: * @param nwid 64-bit network ID */ inline void fromAddress(const Address &ztaddr,uint64_t nwid) - throw() { uint64_t m = ((uint64_t)firstOctetForNetwork(nwid)) << 40; m |= ztaddr.toInt(); // a is 40 bits @@ -208,7 +167,6 @@ public: * @param nwid Network ID */ inline Address toAddress(uint64_t nwid) const - throw() { uint64_t a = _m & 0xffffffffffULL; // least significant 40 bits of MAC are formed from address a ^= ((nwid >> 8) & 0xff) << 32; // ... XORed with bits 8-48 of the nwid in little-endian byte order, so unmask it @@ -224,7 +182,6 @@ public: * @return First octet of MAC for this network */ static inline unsigned char firstOctetForNetwork(uint64_t nwid) - throw() { unsigned char a = ((unsigned char)(nwid & 0xfe) | 0x02); // locally administered, not multicast, from LSB of network ID return ((a == 0x52) ? 0x32 : a); // blacklist 0x52 since it's used by KVM, libvirt, and other popular virtualization engines... seems de-facto standard on Linux @@ -239,29 +196,27 @@ public: /** * @return 6, which is the number of bytes in a MAC, for container compliance */ - inline unsigned int size() const throw() { return 6; } + inline unsigned int size() const { return 6; } - inline unsigned long hashCode() const throw() { return (unsigned long)_m; } + inline unsigned long hashCode() const { return (unsigned long)_m; } inline MAC &operator=(const MAC &m) - throw() { _m = m._m; return *this; } inline MAC &operator=(const uint64_t m) - throw() { _m = m; return *this; } - inline bool operator==(const MAC &m) const throw() { return (_m == m._m); } - inline bool operator!=(const MAC &m) const throw() { return (_m != m._m); } - inline bool operator<(const MAC &m) const throw() { return (_m < m._m); } - inline bool operator<=(const MAC &m) const throw() { return (_m <= m._m); } - inline bool operator>(const MAC &m) const throw() { return (_m > m._m); } - inline bool operator>=(const MAC &m) const throw() { return (_m >= m._m); } + inline bool operator==(const MAC &m) const { return (_m == m._m); } + inline bool operator!=(const MAC &m) const { return (_m != m._m); } + inline bool operator<(const MAC &m) const { return (_m < m._m); } + inline bool operator<=(const MAC &m) const { return (_m <= m._m); } + inline bool operator>(const MAC &m) const { return (_m > m._m); } + inline bool operator>=(const MAC &m) const { return (_m >= m._m); } private: uint64_t _m; diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index 7cbec2e0..f56c675b 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -29,8 +29,6 @@ #include -#include - #include "MAC.hpp" #include "InetAddress.hpp" @@ -94,16 +92,6 @@ public: return MulticastGroup(); } - /** - * @return Human readable string representing this group (MAC/ADI in hex) - */ - inline std::string toString() const - { - char buf[64]; - Utils::ztsnprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi); - return std::string(buf); - } - /** * @return Multicast address */ diff --git a/node/Network.cpp b/node/Network.cpp index bccc0397..f2b6771b 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -51,7 +51,7 @@ namespace ZeroTier { namespace { #ifdef ZT_RULES_ENGINE_DEBUGGING -#define FILTER_TRACE(f,...) { Utils::ztsnprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } +#define FILTER_TRACE(f,...) { snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } static const char *_rtn(const ZT_VirtualNetworkRuleType rt) { switch(rt) { @@ -1257,7 +1257,17 @@ void Network::requestConfiguration(void *tPtr) nconf->rules[13].t = (uint8_t)ZT_NETWORK_RULE_ACTION_DROP; nconf->type = ZT_NETWORK_TYPE_PUBLIC; - Utils::ztsnprintf(nconf->name,sizeof(nconf->name),"adhoc-%.04x-%.04x",(int)startPortRange,(int)endPortRange); + + nconf->name[0] = 'a'; + nconf->name[1] = 'd'; + nconf->name[2] = 'h'; + nconf->name[3] = 'o'; + nconf->name[4] = 'c'; + nconf->name[5] = '-'; + Utils::hex((uint16_t)startPortRange,nconf->name + 6); + nconf->name[10] = '-'; + Utils::hex((uint16_t)endPortRange,nconf->name + 11); + nconf->name[15] = (char)0; this->setConfiguration(tPtr,*nconf,false); delete nconf; diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 65101c3a..e5929923 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -64,7 +64,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (this->staticIps[i].ss_family == AF_INET) { if (v4s.length() > 0) v4s.push_back(','); - v4s.append(this->staticIps[i].toString()); + char buf[64]; + v4s.append(this->staticIps[i].toString(buf)); } } if (v4s.length() > 0) { @@ -75,7 +76,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (this->staticIps[i].ss_family == AF_INET6) { if (v6s.length() > 0) v6s.push_back(','); - v6s.append(this->staticIps[i].toString()); + char buf[64]; + v6s.append(this->staticIps[i].toString(buf)); } } if (v6s.length() > 0) { @@ -94,8 +96,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (ets.length() > 0) ets.push_back(','); char tmp2[16]; - Utils::ztsnprintf(tmp2,sizeof(tmp2),"%x",et); - ets.append(tmp2); + ets.append(Utils::hex((uint16_t)et,tmp2)); } et = 0; } @@ -114,7 +115,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if ((this->specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0) { if (ab.length() > 0) ab.push_back(','); - ab.append(Address(this->specialists[i]).toString().c_str()); + char tmp2[16]; + ab.append(Address(this->specialists[i]).toString(tmp2)); } } if (ab.length() > 0) { diff --git a/node/Node.cpp b/node/Node.cpp index 4b598f61..e28accee 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -82,8 +82,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 if (n > 0) { tmp[n] = (char)0; if (RR->identity.fromString(tmp)) { - RR->publicIdentityStr = RR->identity.toString(false); - RR->secretIdentityStr = RR->identity.toString(true); + RR->identity.toString(false,RR->publicIdentityStr); + RR->identity.toString(true,RR->secretIdentityStr); } else { n = -1; } @@ -92,10 +92,10 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; if (n <= 0) { RR->identity.generate(); - RR->publicIdentityStr = RR->identity.toString(false); - RR->secretIdentityStr = RR->identity.toString(true); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr.data(),(unsigned int)RR->secretIdentityStr.length()); - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr.data(),(unsigned int)RR->publicIdentityStr.length()); + RR->identity.toString(false,RR->publicIdentityStr); + RR->identity.toString(true,RR->secretIdentityStr); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr,(unsigned int)strlen(RR->secretIdentityStr)); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } else { n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { @@ -104,7 +104,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 n = -1; } if (n <= 0) - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr.data(),(unsigned int)RR->publicIdentityStr.length()); + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } try { @@ -386,8 +386,8 @@ uint64_t Node::address() const void Node::status(ZT_NodeStatus *status) const { status->address = RR->identity.address().toInt(); - status->publicIdentity = RR->publicIdentityStr.c_str(); - status->secretIdentity = RR->secretIdentityStr.c_str(); + status->publicIdentity = RR->publicIdentityStr; + status->secretIdentity = RR->secretIdentityStr; status->online = _online ? 1 : 0; } @@ -544,39 +544,6 @@ bool Node::shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,cons return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),localSocket,reinterpret_cast(&remoteAddress)) != 0) : true); } -#ifdef ZT_TRACE -void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) -{ - static Mutex traceLock; - - va_list ap; - char tmp1[1024],tmp2[1024],tmp3[256]; - - Mutex::Lock _l(traceLock); - - time_t now = (time_t)(_now / 1000ULL); -#ifdef __WINDOWS__ - ctime_s(tmp3,sizeof(tmp3),&now); - char *nowstr = tmp3; -#else - char *nowstr = ctime_r(&now,tmp3); -#endif - unsigned long nowstrlen = (unsigned long)strlen(nowstr); - if (nowstr[nowstrlen-1] == '\n') - nowstr[--nowstrlen] = (char)0; - if (nowstr[nowstrlen-1] == '\r') - nowstr[--nowstrlen] = (char)0; - - va_start(ap,fmt); - vsnprintf(tmp2,sizeof(tmp2),fmt,ap); - va_end(ap); - tmp2[sizeof(tmp2)-1] = (char)0; - - Utils::ztsnprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2); - postEvent((void *)0,ZT_EVENT_TRACE,tmp1); -} -#endif // ZT_TRACE - uint64_t Node::prng() { // https://en.wikipedia.org/wiki/Xorshift#xorshift.2B diff --git a/node/Node.hpp b/node/Node.hpp index 55491b06..40903f7c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -195,10 +195,6 @@ public: inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,(int)len); } inline void stateObjectDelete(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2]) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,(const void *)0,-1); } -#ifdef ZT_TRACE - void postTrace(const char *module,unsigned int line,const char *fmt,...); -#endif - bool shouldUsePathForZeroTierTraffic(void *tPtr,const Address &ztaddr,const int64_t localSocket,const InetAddress &remoteAddress); inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 99afe25d..94b96d34 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -27,7 +27,7 @@ #ifndef ZT_RUNTIMEENVIRONMENT_HPP #define ZT_RUNTIMEENVIRONMENT_HPP -#include +#include #include "Constants.hpp" #include "Utils.hpp" @@ -60,11 +60,13 @@ public: ,sa((SelfAwareness *)0) { Utils::getSecureRandom(&instanceId,sizeof(instanceId)); + memset(publicIdentityStr,0,sizeof(publicIdentityStr)); + memset(secretIdentityStr,0,sizeof(secretIdentityStr)); } ~RuntimeEnvironment() { - Utils::burn(reinterpret_cast(const_cast(secretIdentityStr.data())),(unsigned int)secretIdentityStr.length()); + Utils::burn(secretIdentityStr,sizeof(secretIdentityStr)); } /** @@ -77,8 +79,8 @@ public: // This node's identity Identity identity; - std::string publicIdentityStr; - std::string secretIdentityStr; + char publicIdentityStr[ZT_IDENTITY_STRING_BUFFER_LENGTH]; + char secretIdentityStr[ZT_IDENTITY_STRING_BUFFER_LENGTH]; // This is set externally to an instance of this base class NetworkController *localNetworkController; diff --git a/node/Topology.cpp b/node/Topology.cpp index 809bc7e7..e7bbdfae 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -91,12 +91,8 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) { #ifdef ZT_TRACE - if ((!peer)||(peer->address() == RR->identity.address())) { - if (!peer) - fprintf(stderr,"FATAL BUG: addPeer() caught attempt to add NULL peer" ZT_EOL_S); - else fprintf(stderr,"FATAL BUG: addPeer() caught attempt to add peer for self" ZT_EOL_S); - abort(); - } + if ((!peer)||(peer->address() == RR->identity.address())) + return SharedPtr(); #endif SharedPtr np; diff --git a/node/Utils.cpp b/node/Utils.cpp index d2321e16..a3a4c3c3 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -55,90 +55,35 @@ namespace ZeroTier { const char Utils::HEXCHARS[16] = { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; -// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. -static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) -{ - volatile uint8_t *const end = ptr + len; - while (ptr != end) *(ptr++) = (uint8_t)0; -} -static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn; -void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); } - -std::string Utils::hex(const void *data,unsigned int len) +static unsigned long _Utils_itoa(unsigned long n,char *s) { - std::string r; - r.reserve(len * 2); - for(unsigned int i=0;i> 4]); - r.push_back(HEXCHARS[((const unsigned char *)data)[i] & 0x0f]); - } - return r; + if (n == 0) + return 0; + unsigned long pos = _Utils_itoa(n / 10,s); + if (pos >= 22) // sanity check, should be impossible + pos = 22; + s[pos] = '0' + (char)(n % 10); + return pos + 1; } - -std::string Utils::unhex(const char *hex,unsigned int maxlen) +char *Utils::decimal(unsigned long n,char s[24]) { - int n = 1; - unsigned char c,b = 0; - const char *eof = hex + maxlen; - std::string r; - - if (!maxlen) - return r; - - while ((c = (unsigned char)*(hex++))) { - if ((c >= 48)&&(c <= 57)) { // 0..9 - if ((n ^= 1)) - r.push_back((char)(b | (c - 48))); - else b = (c - 48) << 4; - } else if ((c >= 65)&&(c <= 70)) { // A..F - if ((n ^= 1)) - r.push_back((char)(b | (c - (65 - 10)))); - else b = (c - (65 - 10)) << 4; - } else if ((c >= 97)&&(c <= 102)) { // a..f - if ((n ^= 1)) - r.push_back((char)(b | (c - (97 - 10)))); - else b = (c - (97 - 10)) << 4; - } - if (hex == eof) - break; + if (n == 0) { + s[0] = '0'; + s[1] = (char)0; + return s; } - - return r; + s[_Utils_itoa(n,s)] = (char)0; + return s; } -unsigned int Utils::unhex(const char *hex,unsigned int maxlen,void *buf,unsigned int len) +// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. +static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) { - int n = 1; - unsigned char c,b = 0; - unsigned int l = 0; - const char *eof = hex + maxlen; - - if (!maxlen) - return 0; - - while ((c = (unsigned char)*(hex++))) { - if ((c >= 48)&&(c <= 57)) { // 0..9 - if ((n ^= 1)) { - if (l >= len) break; - ((unsigned char *)buf)[l++] = (b | (c - 48)); - } else b = (c - 48) << 4; - } else if ((c >= 65)&&(c <= 70)) { // A..F - if ((n ^= 1)) { - if (l >= len) break; - ((unsigned char *)buf)[l++] = (b | (c - (65 - 10))); - } else b = (c - (65 - 10)) << 4; - } else if ((c >= 97)&&(c <= 102)) { // a..f - if ((n ^= 1)) { - if (l >= len) break; - ((unsigned char *)buf)[l++] = (b | (c - (97 - 10))); - } else b = (c - (97 - 10)) << 4; - } - if (hex == eof) - break; - } - - return l; + volatile uint8_t *const end = ptr + len; + while (ptr != end) *(ptr++) = (uint8_t)0; } +static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn; +void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); } void Utils::getSecureRandom(void *buf,unsigned int bytes) { @@ -244,21 +189,4 @@ bool Utils::scopy(char *dest,unsigned int len,const char *src) return true; } -unsigned int Utils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...) -{ - va_list ap; - - va_start(ap,fmt); - int n = (int)vsnprintf(buf,len,fmt,ap); - va_end(ap); - - if ((n >= (int)len)||(n < 0)) { - if (len) - buf[len - 1] = (char)0; - throw std::length_error("buf[] overflow"); - } - - return (unsigned int)n; -} - } // namespace ZeroTier diff --git a/node/Utils.hpp b/node/Utils.hpp index 212ef247..5a5e9f39 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -70,42 +70,144 @@ public: static void burn(void *ptr,unsigned int len); /** - * Convert binary data to hexadecimal - * - * @param data Data to convert to hex - * @param len Length of data - * @return Hexadecimal string + * @param n Number to convert + * @param s Buffer, at least 24 bytes in size + * @return String containing 'n' in base 10 form */ - static std::string hex(const void *data,unsigned int len); - static inline std::string hex(const std::string &data) { return hex(data.data(),(unsigned int)data.length()); } + static char *decimal(unsigned long n,char s[24]); - /** - * Convert hexadecimal to binary data - * - * This ignores all non-hex characters, just stepping over them and - * continuing. Upper and lower case are supported for letters a-f. - * - * @param hex Hexadecimal ASCII code (non-hex chars are ignored, stops at zero or maxlen) - * @param maxlen Maximum length of hex string buffer - * @return Binary data - */ - static std::string unhex(const char *hex,unsigned int maxlen); - static inline std::string unhex(const std::string &hex) { return unhex(hex.c_str(),(unsigned int)hex.length()); } + static inline char *hex(uint64_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 60) & 0xf]; + s[1] = HEXCHARS[(i >> 56) & 0xf]; + s[2] = HEXCHARS[(i >> 52) & 0xf]; + s[3] = HEXCHARS[(i >> 48) & 0xf]; + s[4] = HEXCHARS[(i >> 44) & 0xf]; + s[5] = HEXCHARS[(i >> 40) & 0xf]; + s[6] = HEXCHARS[(i >> 36) & 0xf]; + s[7] = HEXCHARS[(i >> 32) & 0xf]; + s[8] = HEXCHARS[(i >> 28) & 0xf]; + s[9] = HEXCHARS[(i >> 24) & 0xf]; + s[10] = HEXCHARS[(i >> 20) & 0xf]; + s[11] = HEXCHARS[(i >> 16) & 0xf]; + s[12] = HEXCHARS[(i >> 12) & 0xf]; + s[13] = HEXCHARS[(i >> 8) & 0xf]; + s[14] = HEXCHARS[(i >> 4) & 0xf]; + s[15] = HEXCHARS[i & 0xf]; + s[16] = (char)0; + return s; + } - /** - * Convert hexadecimal to binary data - * - * This ignores all non-hex characters, just stepping over them and - * continuing. Upper and lower case are supported for letters a-f. - * - * @param hex Hexadecimal ASCII - * @param maxlen Maximum length of hex string buffer - * @param buf Buffer to fill - * @param len Length of buffer - * @return Number of characters actually written - */ - static unsigned int unhex(const char *hex,unsigned int maxlen,void *buf,unsigned int len); - static inline unsigned int unhex(const std::string &hex,void *buf,unsigned int len) { return unhex(hex.c_str(),(unsigned int)hex.length(),buf,len); } + static inline char *hex10(uint64_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 36) & 0xf]; + s[1] = HEXCHARS[(i >> 32) & 0xf]; + s[2] = HEXCHARS[(i >> 28) & 0xf]; + s[3] = HEXCHARS[(i >> 24) & 0xf]; + s[4] = HEXCHARS[(i >> 20) & 0xf]; + s[5] = HEXCHARS[(i >> 16) & 0xf]; + s[6] = HEXCHARS[(i >> 12) & 0xf]; + s[7] = HEXCHARS[(i >> 8) & 0xf]; + s[8] = HEXCHARS[(i >> 4) & 0xf]; + s[9] = HEXCHARS[i & 0xf]; + s[10] = (char)0; + return s; + } + + static inline char *hex(uint16_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 12) & 0xf]; + s[1] = HEXCHARS[(i >> 8) & 0xf]; + s[2] = HEXCHARS[(i >> 4) & 0xf]; + s[3] = HEXCHARS[i & 0xf]; + s[4] = (char)0; + return s; + } + + static inline char *hex(uint8_t i,char *const s) + { + s[0] = HEXCHARS[(i >> 4) & 0xf]; + s[1] = HEXCHARS[i & 0xf]; + s[2] = (char)0; + return s; + } + + static inline char *hex(const void *d,unsigned int l,char *s) + { + char *save = s; + for(unsigned int i=0;i(d)[i]; + *(s++) = HEXCHARS[(b >> 4) & 0xf]; + *(s++) = HEXCHARS[b & 0xf]; + } + *s = (char)0; + return save; + } + + static inline unsigned int unhex(const char *h,void *buf,unsigned int buflen) + { + unsigned int l = 0; + while (l < buflen) { + uint8_t hc = (uint8_t)*(h++); + if (!hc) break; + + uint8_t c = 0; + if ((hc >= 48)&&(hc <= 57)) + c = hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c = hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c = hc - 55; + + hc = (uint8_t)*(h++); + if (!hc) break; + + c <<= 4; + if ((hc >= 48)&&(hc <= 57)) + c |= hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c |= hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c |= hc - 55; + + reinterpret_cast(buf)[l++] = c; + } + return l; + } + + static inline unsigned int unhex(const char *h,unsigned int hlen,void *buf,unsigned int buflen) + { + unsigned int l = 0; + const char *hend = h + hlen; + while (l < buflen) { + if (h == hend) break; + uint8_t hc = (uint8_t)*(h++); + if (!hc) break; + + uint8_t c = 0; + if ((hc >= 48)&&(hc <= 57)) + c = hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c = hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c = hc - 55; + + if (h == hend) break; + hc = (uint8_t)*(h++); + if (!hc) break; + + c <<= 4; + if ((hc >= 48)&&(hc <= 57)) + c |= hc - 48; + else if ((hc >= 97)&&(hc <= 102)) + c |= hc - 87; + else if ((hc >= 65)&&(hc <= 70)) + c |= hc - 55; + + reinterpret_cast(buf)[l++] = c; + } + return l; + } /** * Generate secure random bytes @@ -232,20 +334,6 @@ public: */ static bool scopy(char *dest,unsigned int len,const char *src); - /** - * Variant of snprintf that is portable and throws an exception - * - * This just wraps the local implementation whatever it's called, while - * performing a few other checks and adding exceptions for overflow. - * - * @param buf Buffer to write to - * @param len Length of buffer in bytes - * @param fmt Format string - * @param ... Format arguments - * @throws std::length_error buf[] too short (buf[] will still be left null-terminated) - */ - static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...); - /** * Count the number of bits set in an integer * diff --git a/one.cpp b/one.cpp index cbf09121..b1a19e8c 100644 --- a/one.cpp +++ b/one.cpp @@ -260,9 +260,9 @@ static int cli(int argc,char **argv) if (hd) { char p[4096]; #ifdef __APPLE__ - Utils::ztsnprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); + OSUtils::ztsnprintf(p,sizeof(p),"%s/Library/Application Support/ZeroTier/One/authtoken.secret",hd); #else - Utils::ztsnprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); + OSUtils::ztsnprintf(p,sizeof(p),"%s/.zeroTierOneAuthToken",hd); #endif OSUtils::readFile(p,authToken); } @@ -278,7 +278,7 @@ static int cli(int argc,char **argv) InetAddress addr; { char addrtmp[256]; - Utils::ztsnprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); + OSUtils::ztsnprintf(addrtmp,sizeof(addrtmp),"%s/%u",ip.c_str(),port); addr = InetAddress(addrtmp); } @@ -366,7 +366,7 @@ static int cli(int argc,char **argv) std::string addr = path["address"]; const uint64_t now = OSUtils::now(); const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); bestPath = tmp; break; } @@ -378,7 +378,7 @@ static int cli(int argc,char **argv) int64_t vmin = p["versionMinor"]; int64_t vrev = p["versionRev"]; if (vmaj >= 0) { - Utils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + OSUtils::ztsnprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); } else { ver[0] = '-'; ver[1] = (char)0; @@ -527,9 +527,9 @@ static int cli(int argc,char **argv) const uint64_t seed = Utils::hexStrToU64(arg2.c_str()); if ((worldId)&&(seed)) { char jsons[1024]; - Utils::ztsnprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); + OSUtils::ztsnprintf(jsons,sizeof(jsons),"{\"seed\":\"%s\"}",arg2.c_str()); char cl[128]; - Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + OSUtils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -579,11 +579,11 @@ static int cli(int argc,char **argv) if (eqidx != std::string::npos) { if ((arg2.substr(0,eqidx) == "allowManaged")||(arg2.substr(0,eqidx) == "allowGlobal")||(arg2.substr(0,eqidx) == "allowDefault")) { char jsons[1024]; - Utils::ztsnprintf(jsons,sizeof(jsons),"{\"%s\":%s}", + OSUtils::ztsnprintf(jsons,sizeof(jsons),"{\"%s\":%s}", arg2.substr(0,eqidx).c_str(), (((arg2.substr(eqidx,2) == "=t")||(arg2.substr(eqidx,2) == "=1")) ? "true" : "false")); char cl[128]; - Utils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); + OSUtils::ztsnprintf(cl,sizeof(cl),"%u",(unsigned int)strlen(jsons)); requestHeaders["Content-Type"] = "application/json"; requestHeaders["Content-Length"] = cl; unsigned int scode = Http::POST( @@ -648,7 +648,7 @@ static Identity getIdFromArg(char *arg) } else { // identity is to be read from a file std::string idser; if (OSUtils::readFile(arg,idser)) { - if (id.fromString(idser)) + if (id.fromString(idser.c_str())) return id; } } @@ -689,14 +689,15 @@ static int idtool(int argc,char **argv) } } - std::string idser = id.toString(true); + char idtmp[1024]; + std::string idser = id.toString(true,idtmp); if (argc >= 3) { if (!OSUtils::writeFile(argv[2],idser)) { fprintf(stderr,"Error writing to %s" ZT_EOL_S,argv[2]); return 1; } else printf("%s written" ZT_EOL_S,argv[2]); if (argc >= 4) { - idser = id.toString(false); + idser = id.toString(false,idtmp); if (!OSUtils::writeFile(argv[3],idser)) { fprintf(stderr,"Error writing to %s" ZT_EOL_S,argv[3]); return 1; @@ -731,7 +732,8 @@ static int idtool(int argc,char **argv) return 1; } - printf("%s",id.toString(false).c_str()); + char idtmp[1024]; + printf("%s",id.toString(false,idtmp)); } else if (!strcmp(argv[1],"sign")) { if (argc < 4) { idtoolPrintHelp(stdout,argv[0]); @@ -755,7 +757,8 @@ static int idtool(int argc,char **argv) return 1; } C25519::Signature signature = id.sign(inf.data(),(unsigned int)inf.length()); - printf("%s",Utils::hex(signature.data,(unsigned int)signature.size()).c_str()); + char hexbuf[1024]; + printf("%s",Utils::hex(signature.data,(unsigned int)signature.size(),hexbuf)); } else if (!strcmp(argv[1],"verify")) { if (argc < 4) { idtoolPrintHelp(stdout,argv[0]); @@ -774,7 +777,8 @@ static int idtool(int argc,char **argv) return 1; } - std::string signature(Utils::unhex(argv[4])); + char buf[4096]; + std::string signature(buf,Utils::unhex(argv[4],buf,(unsigned int)sizeof(buf))); if ((signature.length() > ZT_ADDRESS_LENGTH)&&(id.verify(inf.data(),(unsigned int)inf.length(),signature.data(),(unsigned int)signature.length()))) { printf("%s signature valid" ZT_EOL_S,argv[3]); } else { @@ -793,14 +797,15 @@ static int idtool(int argc,char **argv) C25519::Pair kp(C25519::generate()); + char idtmp[4096]; nlohmann::json mj; mj["objtype"] = "world"; mj["worldType"] = "moon"; - mj["updatesMustBeSignedBy"] = mj["signingKey"] = Utils::hex(kp.pub.data,(unsigned int)kp.pub.size()); - mj["signingKey_SECRET"] = Utils::hex(kp.priv.data,(unsigned int)kp.priv.size()); - mj["id"] = id.address().toString(); + mj["updatesMustBeSignedBy"] = mj["signingKey"] = Utils::hex(kp.pub.data,(unsigned int)kp.pub.size(),idtmp); + mj["signingKey_SECRET"] = Utils::hex(kp.priv.data,(unsigned int)kp.priv.size(),idtmp); + mj["id"] = id.address().toString(idtmp); nlohmann::json seedj; - seedj["identity"] = id.toString(false); + seedj["identity"] = id.toString(false,idtmp); seedj["stableEndpoints"] = nlohmann::json::array(); (mj["roots"] = nlohmann::json::array()).push_back(seedj); std::string mjd(OSUtils::jsonDump(mj)); @@ -836,9 +841,9 @@ static int idtool(int argc,char **argv) C25519::Pair signingKey; C25519::Public updatesMustBeSignedBy; - Utils::unhex(OSUtils::jsonString(mj["signingKey"],""),signingKey.pub.data,(unsigned int)signingKey.pub.size()); - Utils::unhex(OSUtils::jsonString(mj["signingKey_SECRET"],""),signingKey.priv.data,(unsigned int)signingKey.priv.size()); - Utils::unhex(OSUtils::jsonString(mj["updatesMustBeSignedBy"],""),updatesMustBeSignedBy.data,(unsigned int)updatesMustBeSignedBy.size()); + Utils::unhex(OSUtils::jsonString(mj["signingKey"],"").c_str(),signingKey.pub.data,(unsigned int)signingKey.pub.size()); + Utils::unhex(OSUtils::jsonString(mj["signingKey_SECRET"],"").c_str(),signingKey.priv.data,(unsigned int)signingKey.priv.size()); + Utils::unhex(OSUtils::jsonString(mj["updatesMustBeSignedBy"],"").c_str(),updatesMustBeSignedBy.data,(unsigned int)updatesMustBeSignedBy.size()); std::vector roots; nlohmann::json &rootsj = mj["roots"]; @@ -847,11 +852,11 @@ static int idtool(int argc,char **argv) nlohmann::json &r = rootsj[i]; if (r.is_object()) { roots.push_back(World::Root()); - roots.back().identity = Identity(OSUtils::jsonString(r["identity"],"")); + roots.back().identity = Identity(OSUtils::jsonString(r["identity"],"").c_str()); nlohmann::json &stableEndpointsj = r["stableEndpoints"]; if (stableEndpointsj.is_array()) { for(unsigned long k=0;k<(unsigned long)stableEndpointsj.size();++k) - roots.back().stableEndpoints.push_back(InetAddress(OSUtils::jsonString(stableEndpointsj[k],""))); + roots.back().stableEndpoints.push_back(InetAddress(OSUtils::jsonString(stableEndpointsj[k],"").c_str())); std::sort(roots.back().stableEndpoints.begin(),roots.back().stableEndpoints.end()); } } @@ -864,7 +869,7 @@ static int idtool(int argc,char **argv) Buffer wbuf; w.serialize(wbuf); char fn[128]; - Utils::ztsnprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); + OSUtils::ztsnprintf(fn,sizeof(fn),"%.16llx.moon",w.id()); OSUtils::writeFile(fn,wbuf.data(),wbuf.size()); printf("wrote %s (signed world with timestamp %llu)" ZT_EOL_S,fn,(unsigned long long)now); } diff --git a/osdep/BSDEthernetTap.cpp b/osdep/BSDEthernetTap.cpp index f07f9e5a..8e57d605 100644 --- a/osdep/BSDEthernetTap.cpp +++ b/osdep/BSDEthernetTap.cpp @@ -114,8 +114,8 @@ BSDEthernetTap::BSDEthernetTap( std::vector devFiles(OSUtils::listDirectory("/dev")); for(int i=9993;i<(9993+128);++i) { - Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + OSUtils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); if (std::find(devFiles.begin(),devFiles.end(),std::string(tmpdevname)) == devFiles.end()) { long cpid = (long)vfork(); if (cpid == 0) { @@ -152,8 +152,8 @@ BSDEthernetTap::BSDEthernetTap( /* Other BSDs like OpenBSD only have a limited number of tap devices that cannot be renamed */ for(int i=0;i<64;++i) { - Utils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); - Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); + OSUtils::ztsnprintf(tmpdevname,sizeof(tmpdevname),"tap%d",i); + OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/%s",tmpdevname); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { _dev = tmpdevname; @@ -171,9 +171,9 @@ BSDEthernetTap::BSDEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); + OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + OSUtils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -256,7 +256,8 @@ bool BSDEthernetTap::addIp(const InetAddress &ip) long cpid = (long)vfork(); if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.isV4() ? "inet" : "inet6",ip.toString().c_str(),"alias",(const char *)0); + char tmp[128]; + ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),ip.isV4() ? "inet" : "inet6",ip.toString(tmp),"alias",(const char *)0); ::_exit(-1); } else if (cpid > 0) { int exitcode = -1; @@ -385,7 +386,7 @@ void BSDEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/Http.cpp b/osdep/Http.cpp index 3c556f44..d6d0238c 100644 --- a/osdep/Http.cpp +++ b/osdep/Http.cpp @@ -244,10 +244,10 @@ unsigned int Http::_do( try { char tmp[1024]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); handler.writeBuf.append(tmp); for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) { - Utils::ztsnprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); handler.writeBuf.append(tmp); } handler.writeBuf.append("\r\n"); diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index fc5199f1..c8f9ef9d 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -97,7 +97,7 @@ LinuxEthernetTap::LinuxEthernetTap( char procpath[128],nwids[32]; struct stat sbuf; - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally @@ -134,7 +134,7 @@ LinuxEthernetTap::LinuxEthernetTap( std::map::const_iterator gdmEntry = globalDeviceMap.find(nwids); if (gdmEntry != globalDeviceMap.end()) { Utils::scopy(ifr.ifr_name,sizeof(ifr.ifr_name),gdmEntry->second.c_str()); - Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + OSUtils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); recalledDevice = (stat(procpath,&sbuf) != 0); } @@ -142,8 +142,8 @@ LinuxEthernetTap::LinuxEthernetTap( #ifdef __SYNOLOGY__ int devno = 50; do { - Utils::ztsnprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); - Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + OSUtils::ztsnprintf(ifr.ifr_name,sizeof(ifr.ifr_name),"eth%d",devno++); + OSUtils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); // try zt#++ until we find one that does not exist #else char devno = 0; @@ -158,7 +158,7 @@ LinuxEthernetTap::LinuxEthernetTap( _base32_5_to_8(reinterpret_cast(tmp2) + 5,tmp3 + 10); tmp3[15] = (char)0; memcpy(ifr.ifr_name,tmp3,16); - Utils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + OSUtils::ztsnprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); } while (stat(procpath,&sbuf) == 0); #endif } @@ -264,7 +264,8 @@ static bool ___removeIp(const std::string &_dev,const InetAddress &ip) if (cpid == 0) { OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); - ::execlp("ip","ip","addr","del",ip.toString().c_str(),"dev",_dev.c_str(),(const char *)0); + char iptmp[128]; + ::execlp("ip","ip","addr","del",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0); ::_exit(-1); } else { int exitcode = -1; @@ -296,25 +297,28 @@ bool LinuxEthernetTap::addIpSyn(std::vector ips) // Assemble and write contents of ifcfg-dev file for(int i=0; i<(int)ips.size(); i++) { if (ips[i].isV4()) { + char iptmp[64],iptmp2[64]; std::string numstr4 = ip4_tot > 1 ? std::to_string(ip4) : ""; - cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString() - + "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString()+"\n"; + cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString(iptmp) + + "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; ip4++; } else { + char iptmp[64],iptmp2[64]; std::string numstr6 = ip6_tot > 1 ? std::to_string(ip6) : ""; - cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString() - + "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString()+"\n"; + cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString(iptmp) + + "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString(iptmp2)+"\n"; ip6++; } } OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length()); // Finaly, add IPs for(int i=0; i<(int)ips.size(); i++){ + char iptmp[128],iptmp2[128[; if (ips[i].isV4()) - ::execlp("ip","ip","addr","add",ips[i].toString().c_str(),"broadcast",ips[i].broadcast().toIpString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"broadcast",ips[i].broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0); else - ::execlp("ip","ip","addr","add",ips[i].toString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"dev",_dev.c_str(),(const char *)0); } ::_exit(-1); } else if (cpid > 0) { @@ -345,10 +349,11 @@ bool LinuxEthernetTap::addIp(const InetAddress &ip) if (cpid == 0) { OSUtils::redirectUnixOutputs("/dev/null",(const char *)0); setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1); + char iptmp[128],iptmp2[128]; if (ip.isV4()) { - ::execlp("ip","ip","addr","add",ip.toString().c_str(),"broadcast",ip.broadcast().toIpString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ip.toString(iptmp),"broadcast",ip.broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0); } else { - ::execlp("ip","ip","addr","add",ip.toString().c_str(),"dev",_dev.c_str(),(const char *)0); + ::execlp("ip","ip","addr","add",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0); } ::_exit(-1); } else if (cpid > 0) { diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index fca1c290..3a0b8a7e 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -246,7 +246,6 @@ static std::vector<_RTE> _getRTEs(const InetAddress &target,bool contains) static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *ifscope,const char *localInterface) { - //printf("route %s %s %s %s %s\n",op,target.toString().c_str(),(via) ? via.toString().c_str() : "(null)",(ifscope) ? ifscope : "(null)",(localInterface) ? localInterface : "(null)"); long p = (long)fork(); if (p > 0) { int exitcode = -1; @@ -254,17 +253,19 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress } else if (p == 0) { ::close(STDOUT_FILENO); ::close(STDERR_FILENO); + char ttmp[64]; + char iptmp[64]; if (via) { if ((ifscope)&&(ifscope[0])) { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),via.toIpString().c_str(),(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),via.toIpString(iptmp),(const char *)0); } else { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),via.toIpString().c_str(),(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),via.toIpString(iptmp),(const char *)0); } } else if ((localInterface)&&(localInterface[0])) { if ((ifscope)&&(ifscope[0])) { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),"-interface",localInterface,(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,"-ifscope",ifscope,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),"-interface",localInterface,(const char *)0); } else { - ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString().c_str(),"-interface",localInterface,(const char *)0); + ::execl(ZT_BSD_ROUTE_CMD,ZT_BSD_ROUTE_CMD,op,((target.ss_family == AF_INET6) ? "-inet6" : "-inet"),target.toString(ttmp),"-interface",localInterface,(const char *)0); } } ::_exit(-1); diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 06508e77..882b8255 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -57,6 +57,23 @@ namespace ZeroTier { +unsigned int OSUtils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...) +{ + va_list ap; + + va_start(ap,fmt); + int n = (int)vsnprintf(buf,len,fmt,ap); + va_end(ap); + + if ((n >= (int)len)||(n < 0)) { + if (len) + buf[len - 1] = (char)0; + throw std::length_error("buf[] overflow"); + } + + return (unsigned int)n; +} + #ifdef __UNIX_LIKE__ bool OSUtils::redirectUnixOutputs(const char *stdoutPath,const char *stderrPath) throw() @@ -134,7 +151,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) if (date.QuadPart > 0) { date.QuadPart -= adjust.QuadPart; if ((uint64_t)((date.QuadPart / 10000000) * 1000) < olderThan) { - Utils::ztsnprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); + ztsnprintf(tmp, sizeof(tmp), "%s\\%s", path, ffd.cFileName); if (DeleteFileA(tmp)) ++cleaned; } @@ -157,7 +174,7 @@ long OSUtils::cleanDirectory(const char *path,const uint64_t olderThan) break; if (dptr) { if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_REG)) { - Utils::ztsnprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); + ztsnprintf(tmp,sizeof(tmp),"%s/%s",path,dptr->d_name); if (stat(tmp,&st) == 0) { uint64_t mt = (uint64_t)(st.st_mtime); if ((mt > 0)&&((mt * 1000) < olderThan)) { @@ -464,7 +481,7 @@ std::string OSUtils::jsonString(const nlohmann::json &jv,const char *dfl) return jv; } else if (jv.is_number()) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + ztsnprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); return tmp; } else if (jv.is_boolean()) { return ((bool)jv ? std::string("1") : std::string("0")); @@ -477,9 +494,10 @@ std::string OSUtils::jsonBinFromHex(const nlohmann::json &jv) { std::string s(jsonString(jv,"")); if (s.length() > 0) { - char *buf = new char[(s.length() / 2) + 1]; + unsigned int buflen = (s.length() / 2) + 1; + char *buf = new char[buflen]; try { - unsigned int l = Utils::unhex(s,buf,(unsigned int)s.length()); + unsigned int l = Utils::unhex(s.c_str(),buf,buflen); std::string b(buf,l); delete [] buf; return b; diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index dff7df86..d6f32822 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -33,7 +33,6 @@ #include #include -#include #include #include #include @@ -66,6 +65,20 @@ namespace ZeroTier { class OSUtils { public: + /** + * Variant of snprintf that is portable and throws an exception + * + * This just wraps the local implementation whatever it's called, while + * performing a few other checks and adding exceptions for overflow. + * + * @param buf Buffer to write to + * @param len Length of buffer in bytes + * @param fmt Format string + * @param ... Format arguments + * @throws std::length_error buf[] too short (buf[] will still be left null-terminated) + */ + static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...); + #ifdef __UNIX_LIKE__ /** * Close STDOUT_FILENO and STDERR_FILENO and replace them with output to given path diff --git a/osdep/OSXEthernetTap.cpp b/osdep/OSXEthernetTap.cpp index e082408e..b43d34c0 100644 --- a/osdep/OSXEthernetTap.cpp +++ b/osdep/OSXEthernetTap.cpp @@ -336,7 +336,7 @@ OSXEthernetTap::OSXEthernetTap( char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32]; struct stat stattmp; - Utils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); + OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid); Mutex::Lock _gl(globalTapCreateLock); @@ -391,13 +391,13 @@ OSXEthernetTap::OSXEthernetTap( // Open the first unused tap device if we didn't recall a previous one. if (!recalledDevice) { for(int i=0;i<64;++i) { - Utils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); + OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i); if (stat(devpath,&stattmp)) throw std::runtime_error("no more TAP devices available"); _fd = ::open(devpath,O_RDWR); if (_fd > 0) { char foo[16]; - Utils::ztsnprintf(foo,sizeof(foo),"zt%d",i); + OSUtils::ztsnprintf(foo,sizeof(foo),"zt%d",i); _dev = foo; break; } @@ -413,9 +413,9 @@ OSXEthernetTap::OSXEthernetTap( } // Configure MAC address and MTU, bring interface up - Utils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); - Utils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); - Utils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); + OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]); + OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu); + OSUtils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric); long cpid = (long)vfork(); if (cpid == 0) { ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0); @@ -499,7 +499,8 @@ bool OSXEthernetTap::addIp(const InetAddress &ip) long cpid = (long)vfork(); if (cpid == 0) { - ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toString().c_str(),"alias",(const char *)0); + char tmp[128]; + ::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toString(tmp),"alias",(const char *)0); ::_exit(-1); } else if (cpid > 0) { int exitcode = -1; @@ -519,7 +520,8 @@ bool OSXEthernetTap::removeIp(const InetAddress &ip) if (*i == ip) { long cpid = (long)vfork(); if (cpid == 0) { - execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toIpString().c_str(),"-alias",(const char *)0); + char tmp[128]; + execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toIpString(tmp),"-alias",(const char *)0); _exit(-1); } else if (cpid > 0) { int exitcode = -1; @@ -636,7 +638,7 @@ void OSXEthernetTap::setMtu(unsigned int mtu) long cpid = (long)vfork(); if (cpid == 0) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu); execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0); _exit(-1); } else if (cpid > 0) { diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index df868e7a..b1990486 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -205,7 +205,7 @@ public: memset(externalip,0,sizeof(externalip)); memset(&urls,0,sizeof(urls)); memset(&data,0,sizeof(data)); - Utils::ztsnprintf(inport,sizeof(inport),"%d",localPort); + OSUtils::ztsnprintf(inport,sizeof(inport),"%d",localPort); if ((UPNP_GetValidIGD(devlist,&urls,&data,lanaddr,sizeof(lanaddr)))&&(lanaddr[0])) { #ifdef ZT_PORTMAPPER_TRACE @@ -220,7 +220,7 @@ public: int tryPort = (int)localPort + tries; if (tryPort >= 65535) tryPort = (tryPort - 65535) + 1025; - Utils::ztsnprintf(outport,sizeof(outport),"%u",tryPort); + OSUtils::ztsnprintf(outport,sizeof(outport),"%u",tryPort); // First check and see if this port is already mapped to the // same unique name. If so, keep this mapping and don't try diff --git a/osdep/WindowsEthernetTap.cpp b/osdep/WindowsEthernetTap.cpp index b96ad791..5344268f 100644 --- a/osdep/WindowsEthernetTap.cpp +++ b/osdep/WindowsEthernetTap.cpp @@ -484,7 +484,7 @@ WindowsEthernetTap::WindowsEthernetTap( char tag[24]; // We "tag" registry entries with the network ID to identify persistent devices - Utils::ztsnprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); + OSUtils::ztsnprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)nwid); Mutex::Lock _l(_systemTapInitLock); @@ -601,10 +601,10 @@ WindowsEthernetTap::WindowsEthernetTap( if (_netCfgInstanceId.length() > 0) { char tmps[64]; - unsigned int tmpsl = Utils::ztsnprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; + unsigned int tmpsl = OSUtils::ztsnprintf(tmps,sizeof(tmps),"%.2X-%.2X-%.2X-%.2X-%.2X-%.2X",(unsigned int)mac[0],(unsigned int)mac[1],(unsigned int)mac[2],(unsigned int)mac[3],(unsigned int)mac[4],(unsigned int)mac[5]) + 1; RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"NetworkAddress",REG_SZ,tmps,tmpsl); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MAC",REG_SZ,tmps,tmpsl); - tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); + tmpsl = OSUtils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters,_mySubkeyName.c_str(),"MTU",REG_SZ,tmps,tmpsl); DWORD tmp = 0; @@ -879,7 +879,7 @@ void WindowsEthernetTap::setMtu(unsigned int mtu) HKEY nwAdapters; if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\Class\\{4D36E972-E325-11CE-BFC1-08002BE10318}", 0, KEY_READ | KEY_WRITE, &nwAdapters) == ERROR_SUCCESS) { char tmps[64]; - unsigned int tmpsl = Utils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); + unsigned int tmpsl = OSUtils::ztsnprintf(tmps, sizeof(tmps), "%d", mtu); RegSetKeyValueA(nwAdapters, _mySubkeyName.c_str(), "MTU", REG_SZ, tmps, tmpsl); RegCloseKey(nwAdapters); } @@ -902,7 +902,7 @@ void WindowsEthernetTap::threadMain() HANDLE wait4[3]; OVERLAPPED tapOvlRead,tapOvlWrite; - Utils::ztsnprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); + OSUtils::ztsnprintf(tapPath,sizeof(tapPath),"\\\\.\\Global\\%s.tap",_netCfgInstanceId.c_str()); try { while (_run) { diff --git a/selftest.cpp b/selftest.cpp index ff171aa3..e6705700 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -153,10 +153,11 @@ static int testCrypto() { static unsigned char buf1[16384]; static unsigned char buf2[sizeof(buf1)],buf3[sizeof(buf1)]; + static char hexbuf[1024]; for(int i=0;i<3;++i) { Utils::getSecureRandom(buf1,64); - std::cout << "[crypto] getSecureRandom: " << Utils::hex(buf1,64) << std::endl; + std::cout << "[crypto] getSecureRandom: " << Utils::hex(buf1,64,hexbuf) << std::endl; } std::cout << "[crypto] Testing Salsa20... "; std::cout.flush(); @@ -213,7 +214,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16,hexbuf) << ')' << std::endl; ::free((void *)bb); } @@ -265,7 +266,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16,hexbuf) << ')' << std::endl; ::free((void *)bb); } @@ -427,6 +428,7 @@ static int testIdentity() { Identity id; Buffer<512> buf; + char buf2[1024]; std::cout << "[identity] Validate known-good identity... "; std::cout.flush(); if (!id.fromString(KNOWN_GOOD_IDENTITY)) { @@ -459,7 +461,7 @@ static int testIdentity() uint64_t genstart = OSUtils::now(); id.generate(); uint64_t genend = OSUtils::now(); - std::cout << "(took " << (genend - genstart) << "ms): " << id.toString(true) << std::endl; + std::cout << "(took " << (genend - genstart) << "ms): " << id.toString(true,buf2) << std::endl; std::cout << "[identity] Locally validate identity: "; if (id.locallyValidate()) { std::cout << "PASS" << std::endl; @@ -499,7 +501,7 @@ static int testIdentity() { Identity id2; - id2.fromString(id.toString(true).c_str()); + id2.fromString(id.toString(true,buf2)); std::cout << "[identity] Serialize and deserialize (ASCII w/private): "; if ((id == id2)&&(id2.locallyValidate())) { std::cout << "PASS" << std::endl; @@ -511,7 +513,7 @@ static int testIdentity() { Identity id2; - id2.fromString(id.toString(false).c_str()); + id2.fromString(id.toString(false,buf2)); std::cout << "[identity] Serialize and deserialize (ASCII no private): "; if ((id == id2)&&(id2.locallyValidate())) { std::cout << "PASS" << std::endl; @@ -526,16 +528,18 @@ static int testIdentity() static int testCertificate() { + char buf[4096]; + Identity authority; std::cout << "[certificate] Generating identity to act as authority... "; std::cout.flush(); authority.generate(); - std::cout << authority.address().toString() << std::endl; + std::cout << authority.address().toString(buf) << std::endl; Identity idA,idB; std::cout << "[certificate] Generating identities A and B... "; std::cout.flush(); idA.generate(); idB.generate(); - std::cout << idA.address().toString() << ", " << idB.address().toString() << std::endl; + std::cout << idA.address().toString(buf) << ", " << idB.address().toString(buf) << std::endl; std::cout << "[certificate] Generating certificates A and B..."; CertificateOfMembership cA(10000,100,1,idA.address()); @@ -641,6 +645,8 @@ static void _testExcept(int &depth) static int testOther() { + char buf[1024]; + std::cout << "[other] Testing C++ exceptions... "; std::cout.flush(); int depth = 0; try { @@ -657,6 +663,13 @@ static int testOther() return -1; } + std::cout << "[other] Testing InetAddress encode/decode..."; std::cout.flush(); + std::cout << " " << InetAddress("127.0.0.1/9993").toString(buf); + std::cout << " " << InetAddress("feed:dead:babe:dead:beef:f00d:1234:5678/12345").toString(buf); + std::cout << " " << InetAddress("0/9993").toString(buf); + std::cout << " " << InetAddress("").toString(buf); + std::cout << std::endl; + #if 0 std::cout << "[other] Testing Hashtable... "; std::cout.flush(); { @@ -831,7 +844,7 @@ static int testOther() memset(key, 0, sizeof(key)); memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - Utils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); + OSUtils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); int r = rand() % 128; for(int x=0;xnwid); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",nc->nwid); nj["id"] = tmp; nj["nwid"] = tmp; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)((nc->mac >> 40) & 0xff),(unsigned int)((nc->mac >> 32) & 0xff),(unsigned int)((nc->mac >> 24) & 0xff),(unsigned int)((nc->mac >> 16) & 0xff),(unsigned int)((nc->mac >> 8) & 0xff),(unsigned int)(nc->mac & 0xff)); nj["mac"] = tmp; nj["name"] = nc->name; nj["status"] = nstatus; @@ -223,16 +223,16 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc, nlohmann::json aa = nlohmann::json::array(); for(unsigned int i=0;iassignedAddressCount;++i) { - aa.push_back(reinterpret_cast(&(nc->assignedAddresses[i]))->toString()); + aa.push_back(reinterpret_cast(&(nc->assignedAddresses[i]))->toString(tmp)); } nj["assignedAddresses"] = aa; nlohmann::json ra = nlohmann::json::array(); for(unsigned int i=0;irouteCount;++i) { nlohmann::json rj; - rj["target"] = reinterpret_cast(&(nc->routes[i].target))->toString(); + rj["target"] = reinterpret_cast(&(nc->routes[i].target))->toString(tmp); if (nc->routes[i].via.ss_family == nc->routes[i].target.ss_family) - rj["via"] = reinterpret_cast(&(nc->routes[i].via))->toIpString(); + rj["via"] = reinterpret_cast(&(nc->routes[i].via))->toIpString(tmp); else rj["via"] = nlohmann::json(); rj["flags"] = (int)nc->routes[i].flags; rj["metric"] = (int)nc->routes[i].metric; @@ -252,12 +252,12 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break; } - Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",peer->address); pj["address"] = tmp; pj["versionMajor"] = peer->versionMajor; pj["versionMinor"] = peer->versionMinor; pj["versionRev"] = peer->versionRev; - Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",peer->versionMajor,peer->versionMinor,peer->versionRev); pj["version"] = tmp; pj["latency"] = peer->latency; pj["role"] = prole; @@ -265,7 +265,7 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) nlohmann::json pa = nlohmann::json::array(); for(unsigned int i=0;ipathCount;++i) { nlohmann::json j; - j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(); + j["address"] = reinterpret_cast(&(peer->paths[i].address))->toString(tmp); j["lastSend"] = peer->paths[i].lastSend; j["lastReceive"] = peer->paths[i].lastReceive; j["trustedPathId"] = peer->paths[i].trustedPathId; @@ -280,19 +280,19 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) static void _moonToJson(nlohmann::json &mj,const World &world) { - char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",world.id()); + char tmp[4096]; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",world.id()); mj["id"] = tmp; mj["timestamp"] = world.timestamp(); - mj["signature"] = Utils::hex(world.signature().data,(unsigned int)world.signature().size()); - mj["updatesMustBeSignedBy"] = Utils::hex(world.updatesMustBeSignedBy().data,(unsigned int)world.updatesMustBeSignedBy().size()); + mj["signature"] = Utils::hex(world.signature().data,(unsigned int)world.signature().size(),tmp); + mj["updatesMustBeSignedBy"] = Utils::hex(world.updatesMustBeSignedBy().data,(unsigned int)world.updatesMustBeSignedBy().size(),tmp); nlohmann::json ra = nlohmann::json::array(); for(std::vector::const_iterator r(world.roots().begin());r!=world.roots().end();++r) { nlohmann::json rj; - rj["identity"] = r->identity.toString(false); + rj["identity"] = r->identity.toString(false,tmp); nlohmann::json eps = nlohmann::json::array(); for(std::vector::const_iterator a(r->stableEndpoints.begin());a!=r->stableEndpoints.end();++a) - eps.push_back(a->toString()); + eps.push_back(a->toString(tmp)); rj["stableEndpoints"] = eps; ra.push_back(rj); } @@ -613,7 +613,7 @@ public: json &physical = _localConfig["physical"]; if (physical.is_object()) { for(json::iterator phy(physical.begin());phy!=physical.end();++phy) { - InetAddress net(OSUtils::jsonString(phy.key(),"")); + InetAddress net(OSUtils::jsonString(phy.key(),"").c_str()); if (net) { if (phy.value().is_object()) { uint64_t tpid; @@ -674,7 +674,7 @@ public: // Save primary port to a file so CLIs and GUIs can learn it easily char portstr[64]; - Utils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); + OSUtils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]); OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr)); // Attempt to bind to a secondary port chosen from our ZeroTier address. @@ -712,7 +712,7 @@ public: } if (_ports[2]) { char uniqueName[64]; - Utils::ztsnprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); + OSUtils::ztsnprintf(uniqueName,sizeof(uniqueName),"ZeroTier/%.10llx@%u",_node->address(),_ports[2]); _portMapper = new PortMapper(_ports[2],uniqueName); } } @@ -982,7 +982,7 @@ public: n->second.settings = settings; char nlcpath[4096]; - Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); + OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_networksPath.c_str(),nwid); FILE *out = fopen(nlcpath,"w"); if (out) { fprintf(out,"allowManaged=%d\n",(int)n->second.settings.allowManaged); @@ -1101,7 +1101,7 @@ public: ZT_NodeStatus status; _node->status(&status); - Utils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",status.address); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.10llx",status.address); res["address"] = tmp; res["publicIdentity"] = status.publicIdentity; res["online"] = (bool)(status.online != 0); @@ -1110,7 +1110,7 @@ public: res["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; res["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; res["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; - Utils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%d.%d.%d",ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION); res["version"] = tmp; res["clock"] = OSUtils::now(); @@ -1257,7 +1257,7 @@ public: if ((scode != 200)&&(seed != 0)) { char tmp[64]; - Utils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",id); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",id); res["id"] = tmp; res["roots"] = json::array(); res["timestamp"] = 0; @@ -1395,7 +1395,7 @@ public: json &tryAddrs = v.value()["try"]; if (tryAddrs.is_array()) { for(unsigned long i=0;i 0)) { if (phy.value().is_object()) { if (OSUtils::jsonBool(phy.value()["blacklist"],false)) { @@ -1477,7 +1477,7 @@ public: json &amf = settings["allowManagementFrom"]; if (amf.is_array()) { for(unsigned long i=0;i newManagedIps; @@ -1565,7 +1567,7 @@ public: for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { if (std::find(newManagedIps.begin(),newManagedIps.end(),*ip) == newManagedIps.end()) { if (!n.tap->removeIp(*ip)) - fprintf(stderr,"ERROR: unable to remove ip address %s" ZT_EOL_S, ip->toString().c_str()); + fprintf(stderr,"ERROR: unable to remove ip address %s" ZT_EOL_S, ip->toString(ipbuf)); } } #ifdef __SYNOLOGY__ @@ -1575,7 +1577,7 @@ public: for(std::vector::iterator ip(newManagedIps.begin());ip!=newManagedIps.end();++ip) { if (std::find(n.managedIps.begin(),n.managedIps.end(),*ip) == n.managedIps.end()) { if (!n.tap->addIp(*ip)) - fprintf(stderr,"ERROR: unable to add ip address %s" ZT_EOL_S, ip->toString().c_str()); + fprintf(stderr,"ERROR: unable to add ip address %s" ZT_EOL_S, ip->toString(ipbuf)); } } #endif @@ -1585,7 +1587,7 @@ public: if (syncRoutes) { char tapdev[64]; #ifdef __WINDOWS__ - Utils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); + OSUtils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value); #else Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str()); #endif @@ -1670,7 +1672,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -1851,7 +1853,7 @@ public: &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - Utils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -1919,7 +1921,7 @@ public: if (!n.tap) { try { char friendlyName[128]; - Utils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); + OSUtils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); n.tap = new EthernetTap( _homePath.c_str(), @@ -1933,7 +1935,7 @@ public: *nuptr = (void *)&n; char nlcpath[256]; - Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); std::string nlcbuf; if (OSUtils::readFile(nlcpath,nlcbuf)) { Dictionary<4096> nc; @@ -1954,7 +1956,7 @@ public: while (true) { size_t nextPos = addresses.find(',', pos); std::string address = addresses.substr(pos, (nextPos == std::string::npos ? addresses.size() : nextPos) - pos); - n.settings.allowManagedWhitelist.push_back(InetAddress(address)); + n.settings.allowManagedWhitelist.push_back(InetAddress(address.c_str())); if (nextPos == std::string::npos) break; pos = nextPos + 1; } @@ -2019,7 +2021,7 @@ public: #endif if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) { char nlcpath[256]; - Utils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); + OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); OSUtils::rm(nlcpath); } } else { @@ -2068,20 +2070,20 @@ public: switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); secure = true; break; case ZT_STATE_OBJECT_PLANET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); secure = true; break; default: @@ -2121,19 +2123,19 @@ public: char p[4096]; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.public",_homePath.c_str()); break; case ZT_STATE_OBJECT_IDENTITY_SECRET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); break; case ZT_STATE_OBJECT_PLANET: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - Utils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; default: return -1; @@ -2322,7 +2324,7 @@ public: default: scodestr = "Error"; break; } - Utils::ztsnprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + OSUtils::ztsnprintf(tmpn,sizeof(tmpn),"HTTP/1.1 %.3u %s\r\nCache-Control: no-cache\r\nPragma: no-cache\r\nContent-Type: %s\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", scode, scodestr, contentType.c_str(), diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index e0519827..b4bf03ec 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -284,7 +284,7 @@ bool SoftwareUpdater::check(const uint64_t now) if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; char tmp[512]; - const unsigned int len = Utils::ztsnprintf(tmp,sizeof(tmp), + const unsigned int len = OSUtils::ztsnprintf(tmp,sizeof(tmp), "%c{\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "\":%d," @@ -321,7 +321,8 @@ bool SoftwareUpdater::check(const uint64_t now) // (1) Check the hash itself to make sure the image is basically okay uint8_t sha512[ZT_SHA512_DIGEST_LEN]; SHA512::hash(sha512,_download.data(),(unsigned int)_download.length()); - if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"")) { + char hexbuf[(ZT_SHA512_DIGEST_LEN * 2) + 2]; + if (OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"") == Utils::hex(sha512,ZT_SHA512_DIGEST_LEN,hexbuf)) { // (2) Check signature by signing authority const std::string sig(OSUtils::jsonBinFromHex(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE])); if (Identity(ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY).verify(_download.data(),(unsigned int)_download.length(),sig.data(),(unsigned int)sig.length())) { -- cgit v1.2.3 From f23a43fb81850837c1d85091abf91cc57ae35ef9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 17:32:41 -0700 Subject: More cleanup. --- node/Utils.cpp | 36 +++-------- node/Utils.hpp | 142 +++++++++++++------------------------------- service/SoftwareUpdater.cpp | 33 +++++++++- 3 files changed, 80 insertions(+), 131 deletions(-) (limited to 'node') diff --git a/node/Utils.cpp b/node/Utils.cpp index a3a4c3c3..4ac18366 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -55,6 +55,15 @@ namespace ZeroTier { const char Utils::HEXCHARS[16] = { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; +// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. +static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) +{ + volatile uint8_t *const end = ptr + len; + while (ptr != end) *(ptr++) = (uint8_t)0; +} +static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn; +void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); } + static unsigned long _Utils_itoa(unsigned long n,char *s) { if (n == 0) @@ -76,15 +85,6 @@ char *Utils::decimal(unsigned long n,char s[24]) return s; } -// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers. -static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len) -{ - volatile uint8_t *const end = ptr + len; - while (ptr != end) *(ptr++) = (uint8_t)0; -} -static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn; -void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); } - void Utils::getSecureRandom(void *buf,unsigned int bytes) { static Mutex globalLock; @@ -171,22 +171,4 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) #endif // __WINDOWS__ or not } -bool Utils::scopy(char *dest,unsigned int len,const char *src) -{ - if (!len) - return false; // sanity check - if (!src) { - *dest = (char)0; - return true; - } - char *end = dest + len; - while ((*dest++ = *src++)) { - if (dest == end) { - *(--dest) = (char)0; - return false; - } - } - return true; -} - } // namespace ZeroTier diff --git a/node/Utils.hpp b/node/Utils.hpp index 5a5e9f39..a5b5f7b5 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -228,7 +228,6 @@ public: * @param saveptr Pointer to a char * for temporary reentrant storage */ static inline char *stok(char *str,const char *delim,char **saveptr) - throw() { #ifdef __WINDOWS__ return strtok_s(str,delim,saveptr); @@ -237,30 +236,11 @@ public: #endif } - // String to number converters -- defined here to permit portability - // ifdefs for platforms that lack some of the strtoXX functions. - static inline unsigned int strToUInt(const char *s) - throw() - { - return (unsigned int)strtoul(s,(char **)0,10); - } - static inline int strToInt(const char *s) - throw() - { - return (int)strtol(s,(char **)0,10); - } - static inline unsigned long strToULong(const char *s) - throw() - { - return strtoul(s,(char **)0,10); - } - static inline long strToLong(const char *s) - throw() - { - return strtol(s,(char **)0,10); - } + static inline unsigned int strToUInt(const char *s) { return (unsigned int)strtoul(s,(char **)0,10); } + static inline int strToInt(const char *s) { return (int)strtol(s,(char **)0,10); } + static inline unsigned long strToULong(const char *s) { return strtoul(s,(char **)0,10); } + static inline long strToLong(const char *s) { return strtol(s,(char **)0,10); } static inline unsigned long long strToU64(const char *s) - throw() { #ifdef __WINDOWS__ return (unsigned long long)_strtoui64(s,(char **)0,10); @@ -269,7 +249,6 @@ public: #endif } static inline long long strTo64(const char *s) - throw() { #ifdef __WINDOWS__ return (long long)_strtoi64(s,(char **)0,10); @@ -277,28 +256,11 @@ public: return strtoll(s,(char **)0,10); #endif } - static inline unsigned int hexStrToUInt(const char *s) - throw() - { - return (unsigned int)strtoul(s,(char **)0,16); - } - static inline int hexStrToInt(const char *s) - throw() - { - return (int)strtol(s,(char **)0,16); - } - static inline unsigned long hexStrToULong(const char *s) - throw() - { - return strtoul(s,(char **)0,16); - } - static inline long hexStrToLong(const char *s) - throw() - { - return strtol(s,(char **)0,16); - } + static inline unsigned int hexStrToUInt(const char *s) { return (unsigned int)strtoul(s,(char **)0,16); } + static inline int hexStrToInt(const char *s) { return (int)strtol(s,(char **)0,16); } + static inline unsigned long hexStrToULong(const char *s) { return strtoul(s,(char **)0,16); } + static inline long hexStrToLong(const char *s) { return strtol(s,(char **)0,16); } static inline unsigned long long hexStrToU64(const char *s) - throw() { #ifdef __WINDOWS__ return (unsigned long long)_strtoui64(s,(char **)0,16); @@ -307,7 +269,6 @@ public: #endif } static inline long long hexStrTo64(const char *s) - throw() { #ifdef __WINDOWS__ return (long long)_strtoi64(s,(char **)0,16); @@ -315,11 +276,6 @@ public: return strtoll(s,(char **)0,16); #endif } - static inline double strToDouble(const char *s) - throw() - { - return strtod(s,(char **)0); - } /** * Perform a safe C string copy, ALWAYS null-terminating the result @@ -332,7 +288,23 @@ public: * @param src Source string (if NULL, dest will receive a zero-length string and true is returned) * @return True on success, false on overflow (buffer will still be 0-terminated) */ - static bool scopy(char *dest,unsigned int len,const char *src); + static inline bool scopy(char *dest,unsigned int len,const char *src) + { + if (!len) + return false; // sanity check + if (!src) { + *dest = (char)0; + return true; + } + char *end = dest + len; + while ((*dest++ = *src++)) { + if (dest == end) { + *(--dest) = (char)0; + return false; + } + } + return true; + } /** * Count the number of bits set in an integer @@ -378,14 +350,13 @@ public: } // Byte swappers for big/little endian conversion - static inline uint8_t hton(uint8_t n) throw() { return n; } - static inline int8_t hton(int8_t n) throw() { return n; } - static inline uint16_t hton(uint16_t n) throw() { return htons(n); } - static inline int16_t hton(int16_t n) throw() { return (int16_t)htons((uint16_t)n); } - static inline uint32_t hton(uint32_t n) throw() { return htonl(n); } - static inline int32_t hton(int32_t n) throw() { return (int32_t)htonl((uint32_t)n); } + static inline uint8_t hton(uint8_t n) { return n; } + static inline int8_t hton(int8_t n) { return n; } + static inline uint16_t hton(uint16_t n) { return htons(n); } + static inline int16_t hton(int16_t n) { return (int16_t)htons((uint16_t)n); } + static inline uint32_t hton(uint32_t n) { return htonl(n); } + static inline int32_t hton(int32_t n) { return (int32_t)htonl((uint32_t)n); } static inline uint64_t hton(uint64_t n) - throw() { #if __BYTE_ORDER == __LITTLE_ENDIAN #if defined(__GNUC__) && (!defined(__OpenBSD__)) @@ -406,16 +377,15 @@ public: return n; #endif } - static inline int64_t hton(int64_t n) throw() { return (int64_t)hton((uint64_t)n); } - - static inline uint8_t ntoh(uint8_t n) throw() { return n; } - static inline int8_t ntoh(int8_t n) throw() { return n; } - static inline uint16_t ntoh(uint16_t n) throw() { return ntohs(n); } - static inline int16_t ntoh(int16_t n) throw() { return (int16_t)ntohs((uint16_t)n); } - static inline uint32_t ntoh(uint32_t n) throw() { return ntohl(n); } - static inline int32_t ntoh(int32_t n) throw() { return (int32_t)ntohl((uint32_t)n); } + static inline int64_t hton(int64_t n) { return (int64_t)hton((uint64_t)n); } + + static inline uint8_t ntoh(uint8_t n) { return n; } + static inline int8_t ntoh(int8_t n) { return n; } + static inline uint16_t ntoh(uint16_t n) { return ntohs(n); } + static inline int16_t ntoh(int16_t n) { return (int16_t)ntohs((uint16_t)n); } + static inline uint32_t ntoh(uint32_t n) { return ntohl(n); } + static inline int32_t ntoh(int32_t n) { return (int32_t)ntohl((uint32_t)n); } static inline uint64_t ntoh(uint64_t n) - throw() { #if __BYTE_ORDER == __LITTLE_ENDIAN #if defined(__GNUC__) && !defined(__OpenBSD__) @@ -436,39 +406,7 @@ public: return n; #endif } - static inline int64_t ntoh(int64_t n) throw() { return (int64_t)ntoh((uint64_t)n); } - - /** - * Compare Peer version tuples - * - * @return -1, 0, or 1 based on whether first tuple is less than, equal to, or greater than second - */ - static inline int compareVersion(unsigned int maj1,unsigned int min1,unsigned int rev1,unsigned int b1,unsigned int maj2,unsigned int min2,unsigned int rev2,unsigned int b2) - { - if (maj1 > maj2) - return 1; - else if (maj1 < maj2) - return -1; - else { - if (min1 > min2) - return 1; - else if (min1 < min2) - return -1; - else { - if (rev1 > rev2) - return 1; - else if (rev1 < rev2) - return -1; - else { - if (b1 > b2) - return 1; - else if (b1 < b2) - return -1; - else return 0; - } - } - } - } + static inline int64_t ntoh(int64_t n) { return (int64_t)ntoh((uint64_t)n); } /** * Hexadecimal characters 0-f diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index b4bf03ec..57ecce78 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -57,6 +57,35 @@ namespace ZeroTier { +static int _compareVersion(unsigned int maj1,unsigned int min1,unsigned int rev1,unsigned int b1,unsigned int maj2,unsigned int min2,unsigned int rev2,unsigned int b2) +{ + if (maj1 > maj2) { + return 1; + } else if (maj1 < maj2) { + return -1; + } else { + if (min1 > min2) { + return 1; + } else if (min1 < min2) { + return -1; + } else { + if (rev1 > rev2) { + return 1; + } else if (rev1 < rev2) { + return -1; + } else { + if (b1 > b2) { + return 1; + } else if (b1 < b2) { + return -1; + } else { + return 0; + } + } + } + } +} + SoftwareUpdater::SoftwareUpdater(Node &node,const std::string &homePath) : _node(node), _lastCheckTime(0), @@ -170,7 +199,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void const unsigned int dvMin = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int dvRev = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); const unsigned int dvBld = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); - if (Utils::compareVersion(dvMaj,dvMin,dvRev,dvBld,bestVMaj,bestVMin,bestVRev,bestVBld) > 0) { + if (_compareVersion(dvMaj,dvMin,dvRev,dvBld,bestVMaj,bestVMin,bestVRev,bestVBld) > 0) { latest = &(d->second.meta); bestVMaj = dvMaj; bestVMin = dvMin; @@ -194,7 +223,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void } else { // VERB_LATEST if ((origin == ZT_SOFTWARE_UPDATE_SERVICE)&& - (Utils::compareVersion(rvMaj,rvMin,rvRev,rvBld,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD) > 0)&& + (_compareVersion(rvMaj,rvMin,rvRev,rvBld,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD) > 0)&& (OSUtils::jsonString(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY],"") == ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY)) { const unsigned long len = (unsigned long)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE],0); const std::string hash = OSUtils::jsonBinFromHex(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); -- cgit v1.2.3 From 6fc70f7c16da968a63e69798bbc73ada442cfd2a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 7 Jul 2017 06:50:40 -0700 Subject: More cleanup, Linux build fixes. --- node/C25519.cpp | 290 +++---------------------------------------------- node/Poly1305.cpp | 1 - node/SharedPtr.hpp | 55 +++------- node/Utils.hpp | 22 +++- osdep/Binder.hpp | 4 +- osdep/ManagedRoute.cpp | 9 +- selftest.cpp | 2 +- 7 files changed, 59 insertions(+), 324 deletions(-) (limited to 'node') diff --git a/node/C25519.cpp b/node/C25519.cpp index a78e0466..f35a88c2 100644 --- a/node/C25519.cpp +++ b/node/C25519.cpp @@ -268,9 +268,7 @@ static void recip(unsigned int out[32],const unsigned int z[32]) /* 2^255 - 21 */ mult(out,t1,z11); } -static inline int crypto_scalarmult(unsigned char *q, - const unsigned char *n, - const unsigned char *p) +static inline int crypto_scalarmult(unsigned char *q,const unsigned char *n,const unsigned char *p) { unsigned int work[96]; unsigned char e[32]; @@ -288,12 +286,24 @@ static inline int crypto_scalarmult(unsigned char *q, return 0; } -static const unsigned char base[32] = {9}; - -static inline int crypto_scalarmult_base(unsigned char *q, - const unsigned char *n) +//static const unsigned char base[32] = {9}; +static inline int crypto_scalarmult_base(unsigned char *q,const unsigned char *n) { - return crypto_scalarmult(q,n,base); + //return crypto_scalarmult(q,n,base); + unsigned int work[96]; + unsigned char e[32]; + unsigned int i; + for (i = 0;i < 32;++i) e[i] = n[i]; + e[0] &= 248; + e[31] &= 127; + e[31] |= 64; + for (i = 0;i < 32;++i) work[i] = 9; + mainloop(work,e); + recip(work + 32,work + 32); + mult(work + 64,work,work + 32); + freeze(work + 64); + for (i = 0;i < 32;++i) q[i] = work[64 + i]; + return 0; } ////////////////////////////////////////////////////////////////////////////// @@ -413,20 +423,6 @@ static inline void fe25519_pack(unsigned char r[32], const fe25519 *x) r[i] = y.v[i]; } -#if 0 -static int fe25519_iszero(const fe25519 *x) -{ - int i; - int r; - fe25519 t = *x; - fe25519_freeze(&t); - r = equal(t.v[0],0); - for(i=1;i<32;i++) - r &= equal(t.v[i],0); - return r; -} -#endif - static inline int fe25519_iseq_vartime(const fe25519 *x, const fe25519 *y) { int i; @@ -745,14 +741,6 @@ static inline void sc25519_from32bytes(sc25519 *r, const unsigned char x[32]) barrett_reduce(r, t); } -#if 0 -static void shortsc25519_from16bytes(shortsc25519 *r, const unsigned char x[16]) -{ - int i; - for(i=0;i<16;i++) r->v[i] = x[i]; -} -#endif - static inline void sc25519_from64bytes(sc25519 *r, const unsigned char x[64]) { int i; @@ -761,56 +749,12 @@ static inline void sc25519_from64bytes(sc25519 *r, const unsigned char x[64]) barrett_reduce(r, t); } -#if 0 -static void sc25519_from_shortsc(sc25519 *r, const shortsc25519 *x) -{ - int i; - for(i=0;i<16;i++) - r->v[i] = x->v[i]; - for(i=0;i<16;i++) - r->v[16+i] = 0; -} -#endif - static inline void sc25519_to32bytes(unsigned char r[32], const sc25519 *x) { int i; for(i=0;i<32;i++) r[i] = x->v[i]; } -#if 0 -static int sc25519_iszero_vartime(const sc25519 *x) -{ - int i; - for(i=0;i<32;i++) - if(x->v[i] != 0) return 0; - return 1; -} -#endif - -#if 0 -static int sc25519_isshort_vartime(const sc25519 *x) -{ - int i; - for(i=31;i>15;i--) - if(x->v[i] != 0) return 0; - return 1; -} -#endif - -#if 0 -static int sc25519_lt_vartime(const sc25519 *x, const sc25519 *y) -{ - int i; - for(i=31;i>=0;i--) - { - if(x->v[i] < y->v[i]) return 1; - if(x->v[i] > y->v[i]) return 0; - } - return 0; -} -#endif - static inline void sc25519_add(sc25519 *r, const sc25519 *x, const sc25519 *y) { int i, carry; @@ -824,21 +768,6 @@ static inline void sc25519_add(sc25519 *r, const sc25519 *x, const sc25519 *y) reduce_add_sub(r); } -#if 0 -static void sc25519_sub_nored(sc25519 *r, const sc25519 *x, const sc25519 *y) -{ - crypto_uint32 b = 0; - crypto_uint32 t; - int i; - for(i=0;i<32;i++) - { - t = x->v[i] - y->v[i] - b; - r->v[i] = t & 255; - b = (t >> 8) & 1; - } -} -#endif - static inline void sc25519_mul(sc25519 *r, const sc25519 *x, const sc25519 *y) { int i,j,carry; @@ -860,15 +789,6 @@ static inline void sc25519_mul(sc25519 *r, const sc25519 *x, const sc25519 *y) barrett_reduce(r, t); } -#if 0 -static void sc25519_mul_shortsc(sc25519 *r, const sc25519 *x, const shortsc25519 *y) -{ - sc25519 t; - sc25519_from_shortsc(&t, y); - sc25519_mul(r, x, &t); -} -#endif - static inline void sc25519_window3(signed char r[85], const sc25519 *s) { char carry; @@ -906,45 +826,6 @@ static inline void sc25519_window3(signed char r[85], const sc25519 *s) r[84] += carry; } -#if 0 -static void sc25519_window5(signed char r[51], const sc25519 *s) -{ - char carry; - int i; - for(i=0;i<6;i++) - { - r[8*i+0] = s->v[5*i+0] & 31; - r[8*i+1] = (s->v[5*i+0] >> 5) & 31; - r[8*i+1] ^= (s->v[5*i+1] << 3) & 31; - r[8*i+2] = (s->v[5*i+1] >> 2) & 31; - r[8*i+3] = (s->v[5*i+1] >> 7) & 31; - r[8*i+3] ^= (s->v[5*i+2] << 1) & 31; - r[8*i+4] = (s->v[5*i+2] >> 4) & 31; - r[8*i+4] ^= (s->v[5*i+3] << 4) & 31; - r[8*i+5] = (s->v[5*i+3] >> 1) & 31; - r[8*i+6] = (s->v[5*i+3] >> 6) & 31; - r[8*i+6] ^= (s->v[5*i+4] << 2) & 31; - r[8*i+7] = (s->v[5*i+4] >> 3) & 31; - } - r[8*i+0] = s->v[5*i+0] & 31; - r[8*i+1] = (s->v[5*i+0] >> 5) & 31; - r[8*i+1] ^= (s->v[5*i+1] << 3) & 31; - r[8*i+2] = (s->v[5*i+1] >> 2) & 31; - - /* Making it signed */ - carry = 0; - for(i=0;i<50;i++) - { - r[i] += carry; - r[i+1] += r[i] >> 5; - r[i] &= 31; - carry = r[i] >> 4; - r[i] -= carry<<5; - } - r[50] += carry; -} -#endif - static inline void sc25519_2interleave2(unsigned char r[127], const sc25519 *s1, const sc25519 *s2) { int i; @@ -2052,16 +1933,6 @@ static inline void ge25519_pack(unsigned char r[32], const ge25519_p3 *p) r[31] ^= fe25519_getparity(&tx) << 7; } -#if 0 -static int ge25519_isneutral_vartime(const ge25519_p3 *p) -{ - int ret = 1; - if(!fe25519_iszero(&p->x)) ret = 0; - if(!fe25519_iseq_vartime(&p->y, &p->z)) ret = 0; - return ret; -} -#endif - /* computes [s1]p1 + [s2]p2 */ static void ge25519_double_scalarmult_vartime(ge25519_p3 *r, const ge25519_p3 *p1, const sc25519 *s1, const ge25519_p3 *p2, const sc25519 *s2) { @@ -2137,131 +2008,6 @@ static inline void get_hram(unsigned char *hram, const unsigned char *sm, const SHA512::hash(hram,playground,(unsigned int)smlen); } -// This is the original sign and verify code -- the versions in sign() and -// verify() below the fold are slightly modified in terms of how they behave -// in relation to the message, but the algorithms are the same. - -#if 0 -int crypto_sign_keypair( - unsigned char *pk, - unsigned char *sk - ) -{ - sc25519 scsk; - ge25519 gepk; - unsigned char extsk[64]; - int i; - - randombytes(sk, 32); - crypto_hash_sha512(extsk, sk, 32); - extsk[0] &= 248; - extsk[31] &= 127; - extsk[31] |= 64; - - sc25519_from32bytes(&scsk,extsk); - - ge25519_scalarmult_base(&gepk, &scsk); - ge25519_pack(pk, &gepk); - for(i=0;i<32;i++) - sk[32 + i] = pk[i]; - return 0; -} - -static int crypto_sign( - unsigned char *sm,unsigned long long *smlen, - const unsigned char *m,unsigned long long mlen, - const unsigned char *sk - ) -{ - sc25519 sck, scs, scsk; - ge25519 ger; - unsigned char r[32]; - unsigned char s[32]; - unsigned char extsk[64]; - unsigned long long i; - unsigned char hmg[crypto_hash_sha512_BYTES]; - unsigned char hram[crypto_hash_sha512_BYTES]; - - crypto_hash_sha512(extsk, sk, 32); - extsk[0] &= 248; - extsk[31] &= 127; - extsk[31] |= 64; - - *smlen = mlen+64; - for(i=0;i diff --git a/node/SharedPtr.hpp b/node/SharedPtr.hpp index 09010f67..af66545f 100644 --- a/node/SharedPtr.hpp +++ b/node/SharedPtr.hpp @@ -33,41 +33,19 @@ namespace ZeroTier { /** - * Simple reference counted pointer + * Simple zero-overhead introspective reference counted pointer * * This is an introspective shared pointer. Classes that need to be reference * counted must list this as a 'friend' and must have a private instance of - * AtomicCounter called __refCount. They should also have private destructors, - * since only this class should delete them. - * - * Because this is introspective, it is safe to apply to a naked pointer - * multiple times provided there is always at least one holding SharedPtr. - * - * Once C++11 is ubiquitous, this and a few other things like Thread might get - * torn out for their standard equivalents. + * AtomicCounter called __refCount. */ template class SharedPtr { public: - SharedPtr() - throw() : - _ptr((T *)0) - { - } - - SharedPtr(T *obj) - throw() : - _ptr(obj) - { - ++obj->__refCount; - } - - SharedPtr(const SharedPtr &sp) - throw() : - _ptr(sp._getAndInc()) - { - } + SharedPtr() : _ptr((T *)0) {} + SharedPtr(T *obj) : _ptr(obj) { ++obj->__refCount; } + SharedPtr(const SharedPtr &sp) : _ptr(sp._getAndInc()) {} ~SharedPtr() { @@ -110,21 +88,20 @@ public: * @param with Pointer to swap with */ inline void swap(SharedPtr &with) - throw() { T *tmp = _ptr; _ptr = with._ptr; with._ptr = tmp; } - inline operator bool() const throw() { return (_ptr != (T *)0); } - inline T &operator*() const throw() { return *_ptr; } - inline T *operator->() const throw() { return _ptr; } + inline operator bool() const { return (_ptr != (T *)0); } + inline T &operator*() const { return *_ptr; } + inline T *operator->() const { return _ptr; } /** * @return Raw pointer to held object */ - inline T *ptr() const throw() { return _ptr; } + inline T *ptr() const { return _ptr; } /** * Set this pointer to NULL @@ -162,22 +139,20 @@ public: } } - inline bool operator==(const SharedPtr &sp) const throw() { return (_ptr == sp._ptr); } - inline bool operator!=(const SharedPtr &sp) const throw() { return (_ptr != sp._ptr); } - inline bool operator>(const SharedPtr &sp) const throw() { return (_ptr > sp._ptr); } - inline bool operator<(const SharedPtr &sp) const throw() { return (_ptr < sp._ptr); } - inline bool operator>=(const SharedPtr &sp) const throw() { return (_ptr >= sp._ptr); } - inline bool operator<=(const SharedPtr &sp) const throw() { return (_ptr <= sp._ptr); } + inline bool operator==(const SharedPtr &sp) const { return (_ptr == sp._ptr); } + inline bool operator!=(const SharedPtr &sp) const { return (_ptr != sp._ptr); } + inline bool operator>(const SharedPtr &sp) const { return (_ptr > sp._ptr); } + inline bool operator<(const SharedPtr &sp) const { return (_ptr < sp._ptr); } + inline bool operator>=(const SharedPtr &sp) const { return (_ptr >= sp._ptr); } + inline bool operator<=(const SharedPtr &sp) const { return (_ptr <= sp._ptr); } private: inline T *_getAndInc() const - throw() { if (_ptr) ++_ptr->__refCount; return _ptr; } - T *_ptr; }; diff --git a/node/Utils.hpp b/node/Utils.hpp index a5b5f7b5..1139c9f1 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -76,7 +76,7 @@ public: */ static char *decimal(unsigned long n,char s[24]); - static inline char *hex(uint64_t i,char *const s) + static inline char *hex(uint64_t i,char s[17]) { s[0] = HEXCHARS[(i >> 60) & 0xf]; s[1] = HEXCHARS[(i >> 56) & 0xf]; @@ -98,7 +98,7 @@ public: return s; } - static inline char *hex10(uint64_t i,char *const s) + static inline char *hex10(uint64_t i,char s[11]) { s[0] = HEXCHARS[(i >> 36) & 0xf]; s[1] = HEXCHARS[(i >> 32) & 0xf]; @@ -114,7 +114,21 @@ public: return s; } - static inline char *hex(uint16_t i,char *const s) + static inline char *hex(uint32_t i,char s[9]) + { + s[0] = HEXCHARS[(i >> 28) & 0xf]; + s[1] = HEXCHARS[(i >> 24) & 0xf]; + s[2] = HEXCHARS[(i >> 20) & 0xf]; + s[3] = HEXCHARS[(i >> 16) & 0xf]; + s[4] = HEXCHARS[(i >> 12) & 0xf]; + s[5] = HEXCHARS[(i >> 8) & 0xf]; + s[6] = HEXCHARS[(i >> 4) & 0xf]; + s[7] = HEXCHARS[i & 0xf]; + s[8] = (char)0; + return s; + } + + static inline char *hex(uint16_t i,char s[5]) { s[0] = HEXCHARS[(i >> 12) & 0xf]; s[1] = HEXCHARS[(i >> 8) & 0xf]; @@ -124,7 +138,7 @@ public: return s; } - static inline char *hex(uint8_t i,char *const s) + static inline char *hex(uint8_t i,char s[3]) { s[0] = HEXCHARS[(i >> 4) & 0xf]; s[1] = HEXCHARS[i & 0xf]; diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index f956a67e..126dba28 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -176,7 +176,7 @@ public: const unsigned long pid = (unsigned long)getpid(); // Get all device names - Utils::ztsnprintf(fn,sizeof(fn),"/proc/%lu/net/dev",pid); + OSUtils::ztsnprintf(fn,sizeof(fn),"/proc/%lu/net/dev",pid); FILE *procf = fopen(fn,"r"); if (procf) { while (fgets(tmp,sizeof(tmp),procf)) { @@ -192,7 +192,7 @@ public: } // Get IPv6 addresses (and any device names we don't already know) - Utils::ztsnprintf(fn,sizeof(fn),"/proc/%lu/net/if_inet6",pid); + OSUtils::ztsnprintf(fn,sizeof(fn),"/proc/%lu/net/if_inet6",pid); procf = fopen(fn,"r"); if (procf) { while (fgets(tmp,sizeof(tmp),procf)) { diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 3a0b8a7e..5b5bf541 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -286,12 +286,13 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress } else if (p == 0) { ::close(STDOUT_FILENO); ::close(STDERR_FILENO); + char ipbuf[64],ipbuf2[64]; if (via) { - ::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"via",via.toIpString().c_str(),(const char *)0); - ::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"via",via.toIpString().c_str(),(const char *)0); + ::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"via",via.toIpString(ipbuf2),(const char *)0); + ::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"via",via.toIpString(ipbuf2),(const char *)0); } else if ((localInterface)&&(localInterface[0])) { - ::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"dev",localInterface,(const char *)0); - ::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"dev",localInterface,(const char *)0); + ::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"dev",localInterface,(const char *)0); + ::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"dev",localInterface,(const char *)0); } ::_exit(-1); } diff --git a/selftest.cpp b/selftest.cpp index e6705700..882422bc 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -844,7 +844,7 @@ static int testOther() memset(key, 0, sizeof(key)); memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - OSUtils::ztsnprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); + Utils::hex((uint32_t)((rand() % 1000) + (q * 1000)),key[q]); int r = rand() % 128; for(int x=0;x Date: Fri, 7 Jul 2017 16:58:05 -0700 Subject: Remote trace: plumbing, replace old TRACE with calls to Trace object. --- include/ZeroTierOne.h | 9 +- node/IncomingPacket.cpp | 180 ++++++++++------------------- node/Membership.cpp | 29 ++--- node/Multicaster.cpp | 4 - node/Network.cpp | 274 +++++++++----------------------------------- node/NetworkConfig.cpp | 2 + node/NetworkConfig.hpp | 7 ++ node/Node.cpp | 3 + node/Node.hpp | 7 -- node/OutboundMulticast.cpp | 13 --- node/Packet.cpp | 44 ------- node/Packet.hpp | 33 ++++-- node/Peer.cpp | 57 ++++----- node/RuntimeEnvironment.hpp | 3 +- node/SelfAwareness.cpp | 3 +- node/Switch.cpp | 71 +++--------- node/Topology.cpp | 10 +- node/Topology.hpp | 6 - node/Trace.cpp | 197 +++++++++++++++++++++++++++++++ node/Trace.hpp | 157 +++++++++++++++++++++++++ objects.mk | 1 + 21 files changed, 577 insertions(+), 533 deletions(-) create mode 100644 node/Trace.cpp create mode 100644 node/Trace.hpp (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 1365a9a0..f7681768 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -29,8 +29,8 @@ * engine. */ -#ifndef ZT_ZEROTIERONE_H -#define ZT_ZEROTIERONE_H +#ifndef ZT_ZEROTIER_API_H +#define ZT_ZEROTIER_API_H #include @@ -92,6 +92,11 @@ extern "C" { */ #define ZT_MAX_MTU 10000 +/** + * Maximum size of a remote trace message's serialized Dictionary + */ +#define ZT_MAX_REMOTE_TRACE_SIZE 10000 + /** * Maximum length of network short name */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index ac8514c6..e1fb180c 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -48,6 +48,7 @@ #include "Capability.hpp" #include "Tag.hpp" #include "Revocation.hpp" +#include "Trace.hpp" namespace ZeroTier { @@ -63,11 +64,12 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) // If this is marked as a packet via a trusted path, check source address and path ID. // Obviously if no trusted paths are configured this always returns false and such // packets are dropped on the floor. - if (RR->topology->shouldInboundPathBeTrusted(_path->address(),trustedPathId())) { + const uint64_t tpid = trustedPathId(); + if (RR->topology->shouldInboundPathBeTrusted(_path->address(),tpid)) { + RR->t->incomingPacketTrustedPath(_path,packetId(),sourceAddress,tpid,true); trusted = true; - TRACE("TRUSTED PATH packet approved from %s(%s), trusted path ID %llx",sourceAddress.toString().c_str(),_path->address().toString().c_str(),trustedPathId()); } else { - TRACE("dropped packet from %s(%s), cipher set to trusted path mode but path %llx@%s is not trusted!",sourceAddress.toString().c_str(),_path->address().toString().c_str(),trustedPathId(),_path->address().toString().c_str()); + RR->t->incomingPacketTrustedPath(_path,packetId(),sourceAddress,tpid,false); return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { @@ -80,19 +82,18 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) if (!trusted) { if (!dearmor(peer->key())) { //fprintf(stderr,"dropped packet from %s(%s), MAC authentication failed (size: %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); - TRACE("dropped packet from %s(%s), MAC authentication failed (size: %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); + RR->t->incomingPacketMessageAuthenticationFailure(_path,packetId(),sourceAddress); return true; } } if (!uncompress()) { //fprintf(stderr,"dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); - TRACE("dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); + RR->t->incomingPacketInvalid(_path,packetId(),sourceAddress,Packet::VERB_NOP,"LZ4 decompression failed"); return true; } const Packet::Verb v = verb(); - //TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_path->address().toString().c_str()); switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" @@ -121,9 +122,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) return false; } } catch ( ... ) { - // Exceptions are more informatively caught in _do...() handlers but - // this outer try/catch will catch anything else odd. - TRACE("dropped ??? from %s(%s): unexpected exception in tryDecode()",sourceAddress.toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),sourceAddress,Packet::VERB_NOP,"unexpected exception in tryDecode() (outer)"); return true; } } @@ -135,8 +134,6 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar const uint64_t inRePacketId = at(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID); const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE]; - //TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); - /* Security note: we do not gate doERROR() with expectingReplyTo() to * avoid having to log every outgoing packet ID. Instead we put the * logic to determine whether we should consider an ERROR in each @@ -192,7 +189,6 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->gate(tPtr,peer))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); - TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str()); RR->mc->remove(network->id(),mg,peer->address()); } } break; @@ -202,7 +198,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); } catch ( ... ) { - TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_ERROR,"unexpected exception"); } return true; } @@ -223,11 +219,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); if (protoVersion < ZT_PROTO_VERSION_MIN) { - TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"protocol version too old"); return true; } if (fromAddress != id.address()) { - TRACE("dropped HELLO from %s(%s): identity does not match packet source address",fromAddress.toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"identity/address mismatch"); return true; } @@ -245,7 +241,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { if (dearmor(key)) { // ensure packet is authentic, otherwise drop - TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"address collision"); Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)Packet::VERB_HELLO); outp.append((uint64_t)pid); @@ -253,10 +249,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool outp.armor(key,true,_path->nextOutgoingCounter()); _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } else { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); } } else { - TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); } return true; @@ -264,7 +260,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool // Identity is the same as the one we already have -- check packet integrity if (!dearmor(peer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); return true; } @@ -276,24 +272,26 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool // Sanity check: this basically can't happen if (alreadyAuthenticated) { - TRACE("dropped HELLO from %s(%s): somehow already authenticated with unknown peer?",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"illegal alreadyAuthenticated state"); return true; } // Check rate limits - if (!RR->node->rateGateIdentityVerification(now,_path->address())) + if (!RR->node->rateGateIdentityVerification(now,_path->address())) { + RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"rate limit exceeded"); return true; + } // Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap) SharedPtr newPeer(new Peer(RR,RR->identity,id)); if (!dearmor(newPeer->key())) { - TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); return true; } // Check that identity's address is valid as per the derivation function if (!id.locallyValidate()) { - TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"invalid identity"); return true; } @@ -418,7 +416,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_HELLO,"unexpected exception"); } return true; } @@ -429,12 +427,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); - if (!RR->node->expectingReplyTo(inRePacketId)) { - TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId()); + if (!RR->node->expectingReplyTo(inRePacketId)) return true; - } - - //TRACE("%s(%s): OK(%s)",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb)); switch(inReVerb) { @@ -447,11 +441,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION]; const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION]; const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION); - - if (vProto < ZT_PROTO_VERSION_MIN) { - TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_path->address().toString().c_str()); + if (vProto < ZT_PROTO_VERSION_MIN) return true; - } InetAddress externalSurfaceAddress; unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2; @@ -484,12 +475,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP } else ptr += 2; } -#ifdef ZT_TRACE - const std::string tmp1(source().toString()); - const std::string tmp2(_path->address().toString()); - TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u",tmp1.c_str(),tmp2.c_str(),vMajor,vMinor,vRevision,latency); -#endif - if (!hops()) peer->addDirectLatencyMeasurment((unsigned int)latency); peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); @@ -516,7 +501,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP const SharedPtr network(RR->node->network(nwid)); if (network) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); - //TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size()); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); } @@ -527,8 +511,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID); const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI)); - //TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags); - const SharedPtr network(RR->node->network(nwid)); if (network) { unsigned int offset = 0; @@ -555,7 +537,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); } catch ( ... ) { - TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_OK,"unexpected exception"); } return true; } @@ -563,10 +545,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { - if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) { - TRACE("dropped WHOIS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) return true; - } Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); outp.append((unsigned char)Packet::VERB_WHOIS); @@ -595,7 +575,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_WHOIS,"unexpected exception"); } return true; } @@ -603,9 +583,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { - if (!RR->topology->isUpstream(peer->identity())) { - TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str()); - } else { + if (RR->topology->isUpstream(peer->identity())) { const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); const SharedPtr rendezvousWith(RR->topology->getPeer(tPtr,with)); if (rendezvousWith) { @@ -614,22 +592,16 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localSocket(),atAddr)) { - RR->node->putPacket(tPtr,_path->localSocket(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + const uint64_t junk = RR->node->prng(); + RR->node->putPacket(tPtr,_path->localSocket(),atAddr,&junk,4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls rendezvousWith->attemptToContactAt(tPtr,_path->localSocket(),atAddr,RR->node->now(),false,0); - TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); - } else { - TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str()); } - } else { - TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str()); } - } else { - TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str()); } } peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_RENDEZVOUS,"unexpected exception"); } return true; } @@ -652,16 +624,15 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); } } else { - TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + RR->t->networkAccessDenied(network,_path,packetId(),size(),peer->address(),Packet::VERB_FRAME,true); } } else { - TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { - TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_FRAME,"unexpected exception"); } return true; } @@ -683,7 +654,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const } if (!network->gate(tPtr,peer)) { - TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id()); + RR->t->networkAccessDenied(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,true); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); return true; @@ -696,8 +667,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); - if ((!from)||(from.isMulticast())||(from == network->mac())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str()); + if ((!from)||(from == network->mac())) { peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -708,19 +678,19 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (to.isMulticast()) { if (network->config().multicastLimit == 0) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: network %.16llx does not allow multicast",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } else if (!network->config().permitsBridging(RR->identity.address())) { - TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -743,12 +713,10 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); } else { - TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)); - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { - TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_EXT_FRAME,"unexpected exception"); } return true; } @@ -756,10 +724,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { - if (!peer->rateGateEchoRequest(RR->node->now())) { - TRACE("dropped ECHO from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + if (!peer->rateGateEchoRequest(RR->node->now())) return true; - } const uint64_t pid = packetId(); Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); @@ -772,7 +738,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_ECHO,"unexpected exception"); } return true; } @@ -820,7 +786,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,c peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { - TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_LIKE,"unexpected exception"); } return true; } @@ -828,10 +794,8 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,c bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { - if (!peer->rateGateCredentialsReceived(RR->node->now())) { - TRACE("dropped NETWORK_CREDENTIALS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); + if (!peer->rateGateCredentialsReceived(RR->node->now())) return true; - } CertificateOfMembership com; Capability cap; @@ -942,12 +906,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t } peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); - } catch (std::exception &exc) { - //fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); - TRACE("dropped NETWORK_CREDENTIALS from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { - //fprintf(stderr,"dropped NETWORK_CREDENTIALS from %s(%s): unknown exception" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str()); - TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_NETWORK_CREDENTIALS,"unexpected exception"); } return true; } @@ -975,12 +935,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void } peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); - } catch (std::exception &exc) { - //fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): %s" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str(),exc.what()); - TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what()); } catch ( ... ) { - //fprintf(stderr,"dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception" ZT_EOL_S,source().toString().c_str(),_path->address().toString().c_str()); - TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_NETWORK_CONFIG_REQUEST,"unexpected exception"); } return true; } @@ -1003,7 +959,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,c } peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_NETWORK_CONFIG,"unexpected exception"); } return true; } @@ -1016,8 +972,6 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); const unsigned int gatherLimit = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); - //TRACE("<address().toString().c_str(),gatherLimit,nwid,mg.toString().c_str()); - const SharedPtr network(RR->node->network(nwid)); if ((flags & 0x01) != 0) { @@ -1029,9 +983,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr network->addCredential(tPtr,com); else RR->mc->addCredential(tPtr,com,false); } - } catch ( ... ) { - TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str()); - } + } catch ( ... ) {} // discard invalid COMs } const bool trustEstablished = ((network)&&(network->gate(tPtr,peer))); @@ -1053,7 +1005,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); } catch ( ... ) { - TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_GATHER,"unexpected exception"); } return true; } @@ -1078,18 +1030,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, } if (!network->gate(tPtr,peer)) { - TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); + RR->t->networkAccessDenied(network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,true); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); return true; } - if (network->config().multicastLimit == 0) { - TRACE("dropped MULTICAST_FRAME from %s(%s): network %.16llx does not allow multicast",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); - return true; - } - unsigned int gatherLimit = 0; if ((flags & 0x02) != 0) { gatherLimit = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT); @@ -1108,16 +1054,20 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, const unsigned int etherType = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE); const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); - //TRACE("<address().toString().c_str(),flags,frameLen); + if (network->config().multicastLimit == 0) { + RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + return true; + } if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) { if (!to.mac().isMulticast()) { - TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_FRAME,"destination not multicast"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { - TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } @@ -1126,16 +1076,15 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id()); + RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay return true; } } const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) { + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); - } } if (gatherLimit) { @@ -1158,7 +1107,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); } } catch ( ... ) { - TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_FRAME,"unexpected exception"); } return true; } @@ -1170,7 +1119,6 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { - //TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str()); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); return true; } @@ -1202,10 +1150,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { peer->redirect(tPtr,_path->localSocket(),a,now); } else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { - TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); - } else { - //TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } } } break; @@ -1219,10 +1164,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { peer->redirect(tPtr,_path->localSocket(),a,now); } else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { - TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); - } else { - //TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str()); } } } break; @@ -1232,7 +1174,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_PUSH_DIRECT_PATHS,"unexpected exception"); } return true; } @@ -1250,7 +1192,7 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con } peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false); } catch ( ... ) { - TRACE("dropped USER_MESSAGE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str()); + RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_USER_MESSAGE,"unexpected exception"); } return true; } diff --git a/node/Membership.cpp b/node/Membership.cpp index 466f9021..be6ea6a5 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -33,6 +33,7 @@ #include "Switch.hpp" #include "Packet.hpp" #include "Node.hpp" +#include "Trace.hpp" #define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 3) @@ -128,27 +129,25 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme { const uint64_t newts = com.timestamp(); if (newts <= _comRevocationThreshold) { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (revoked)",com.issuedTo().toString().c_str(),com.networkId()); + RR->t->credentialRejected(com,"revoked"); return ADD_REJECTED; } const uint64_t oldts = _com.timestamp(); if (newts < oldts) { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (older than current)",com.issuedTo().toString().c_str(),com.networkId()); + RR->t->credentialRejected(com,"old"); return ADD_REJECTED; } - if ((newts == oldts)&&(_com == com)) { - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (redundant)",com.issuedTo().toString().c_str(),com.networkId()); + if ((newts == oldts)&&(_com == com)) return ADD_ACCEPTED_REDUNDANT; - } switch(com.verify(RR,tPtr)) { default: - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (invalid signature or object)",com.issuedTo().toString().c_str(),com.networkId()); + RR->t->credentialRejected(com,"invalid"); return ADD_REJECTED; case 0: - TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId()); _com = com; + RR->t->credentialAccepted(com); return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -162,27 +161,25 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot C *rc = remoteCreds.get(cred.id()); if (rc) { if (rc->timestamp() > cred.timestamp()) { - TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (older than credential we have)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + RR->t->credentialRejected(cred,"old"); return Membership::ADD_REJECTED; } - if (*rc == cred) { - //TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (redundant)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + if (*rc == cred) return Membership::ADD_ACCEPTED_REDUNDANT; - } } const uint64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); if ((rt)&&(*rt >= cred.timestamp())) { - TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (timestamp below revocation threshold)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + RR->t->credentialRejected(cred,"revoked"); return Membership::ADD_REJECTED; } switch(cred.verify(RR,tPtr)) { default: - TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (invalid)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + RR->t->credentialRejected(cred,"invalid"); return Membership::ADD_REJECTED; case 0: - TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (new)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + RR->t->credentialAccepted(cred); if (!rc) rc = &(remoteCreds[cred.id()]); *rc = cred; @@ -201,12 +198,14 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme uint64_t *rt; switch(rev.verify(RR,tPtr)) { default: + RR->t->credentialRejected(rev,"invalid"); return ADD_REJECTED; case 0: { const Credential::Type ct = rev.type(); switch(ct) { case Credential::CREDENTIAL_TYPE_COM: if (rev.threshold() > _comRevocationThreshold) { + RR->t->credentialAccepted(rev); _comRevocationThreshold = rev.threshold(); return ADD_ACCEPTED_NEW; } @@ -217,10 +216,12 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme rt = &(_revocations[credentialKey(ct,rev.credentialId())]); if (*rt < rev.threshold()) { *rt = rev.threshold(); + _comRevocationThreshold = rev.threshold(); return ADD_ACCEPTED_NEW; } return ADD_ACCEPTED_REDUNDANT; default: + RR->t->credentialRejected(rev,"invalid"); return ADD_REJECTED; } } diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 52213364..fb7b068f 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -139,8 +139,6 @@ restart_member_scan: appendTo.setAt(totalAt,(uint32_t)totalKnown); appendTo.setAt(addedAt,(uint16_t)added); - //TRACE("..MC Multicaster::gather() attached %u of %u peers for %.16llx/%s (2)",n,(unsigned int)(gs->second.members.size() - skipped),nwid,mg.toString().c_str()); - return added; } @@ -386,8 +384,6 @@ void Multicaster::_add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGrou gs.members.push_back(MulticastGroupMember(member,now)); - //TRACE("..MC %s joined multicast group %.16llx/%s via %s",member.toString().c_str(),nwid,mg.toString().c_str(),((learnedFrom) ? learnedFrom.toString().c_str() : "(direct)")); - for(std::list::iterator tx(gs.txQueue.begin());tx!=gs.txQueue.end();) { if (tx->atLimit()) gs.txQueue.erase(tx++); diff --git a/node/Network.cpp b/node/Network.cpp index f2b6771b..575b0170 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -42,89 +42,12 @@ #include "NetworkController.hpp" #include "Node.hpp" #include "Peer.hpp" - -// Uncomment to make the rules engine dump trace info to stdout -//#define ZT_RULES_ENGINE_DEBUGGING 1 +#include "Trace.hpp" namespace ZeroTier { namespace { -#ifdef ZT_RULES_ENGINE_DEBUGGING -#define FILTER_TRACE(f,...) { snprintf(dpbuf,sizeof(dpbuf),f,##__VA_ARGS__); dlog.push_back(std::string(dpbuf)); } -static const char *_rtn(const ZT_VirtualNetworkRuleType rt) -{ - switch(rt) { - case ZT_NETWORK_RULE_ACTION_DROP: return "ACTION_DROP"; - case ZT_NETWORK_RULE_ACTION_ACCEPT: return "ACTION_ACCEPT"; - case ZT_NETWORK_RULE_ACTION_TEE: return "ACTION_TEE"; - case ZT_NETWORK_RULE_ACTION_WATCH: return "ACTION_WATCH"; - case ZT_NETWORK_RULE_ACTION_REDIRECT: return "ACTION_REDIRECT"; - case ZT_NETWORK_RULE_ACTION_BREAK: return "ACTION_BREAK"; - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: return "MATCH_SOURCE_ZEROTIER_ADDRESS"; - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: return "MATCH_DEST_ZEROTIER_ADDRESS"; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: return "MATCH_VLAN_ID"; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: return "MATCH_VLAN_PCP"; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: return "MATCH_VLAN_DEI"; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: return "MATCH_MAC_SOURCE"; - case ZT_NETWORK_RULE_MATCH_MAC_DEST: return "MATCH_MAC_DEST"; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: return "MATCH_IPV4_SOURCE"; - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: return "MATCH_IPV4_DEST"; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: return "MATCH_IPV6_SOURCE"; - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: return "MATCH_IPV6_DEST"; - case ZT_NETWORK_RULE_MATCH_IP_TOS: return "MATCH_IP_TOS"; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: return "MATCH_IP_PROTOCOL"; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: return "MATCH_ETHERTYPE"; - case ZT_NETWORK_RULE_MATCH_ICMP: return "MATCH_ICMP"; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: return "MATCH_IP_SOURCE_PORT_RANGE"; - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: return "MATCH_IP_DEST_PORT_RANGE"; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: return "MATCH_CHARACTERISTICS"; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: return "MATCH_FRAME_SIZE_RANGE"; - case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: return "MATCH_TAGS_DIFFERENCE"; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: return "MATCH_TAGS_BITWISE_AND"; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: return "MATCH_TAGS_BITWISE_OR"; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: return "MATCH_TAGS_BITWISE_XOR"; - default: return "???"; - } -} -static const void _dumpFilterTrace(const char *ruleName,uint8_t thisSetMatches,bool inbound,const Address &ztSource,const Address &ztDest,const MAC &macSource,const MAC &macDest,const std::vector &dlog,unsigned int frameLen,unsigned int etherType,const char *msg) -{ - static volatile unsigned long cnt = 0; - printf("%.6lu %c %s %s frameLen=%u etherType=%u" ZT_EOL_S, - cnt++, - ((thisSetMatches) ? 'Y' : '.'), - ruleName, - ((inbound) ? "INBOUND" : "OUTBOUND"), - frameLen, - etherType - ); - for(std::vector::const_iterator m(dlog.begin());m!=dlog.end();++m) - printf(" | %s" ZT_EOL_S,m->c_str()); - printf(" + %c %s->%s %.2x:%.2x:%.2x:%.2x:%.2x:%.2x->%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" ZT_EOL_S, - ((thisSetMatches) ? 'Y' : '.'), - ztSource.toString().c_str(), - ztDest.toString().c_str(), - (unsigned int)macSource[0], - (unsigned int)macSource[1], - (unsigned int)macSource[2], - (unsigned int)macSource[3], - (unsigned int)macSource[4], - (unsigned int)macSource[5], - (unsigned int)macDest[0], - (unsigned int)macDest[1], - (unsigned int)macDest[2], - (unsigned int)macDest[3], - (unsigned int)macDest[4], - (unsigned int)macDest[5] - ); - if (msg) - printf(" + (%s)" ZT_EOL_S,msg); - fflush(stdout); -} -#else -#define FILTER_TRACE(f,...) {} -#endif // ZT_RULES_ENGINE_DEBUGGING - // Returns true if packet appears valid; pos and proto will be set static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto) { @@ -162,8 +85,10 @@ enum _doZtFilterResult DOZTFILTER_ACCEPT, DOZTFILTER_SUPER_ACCEPT }; + static _doZtFilterResult _doZtFilter( const RuntimeEnvironment *RR, + Trace::RuleResultLog &rrl, const NetworkConfig &nconf, const Membership *membership, // can be NULL const bool inbound, @@ -181,11 +106,6 @@ static _doZtFilterResult _doZtFilter( unsigned int &ccLength, // MUTABLE -- set to length of packet payload to TEE bool &ccWatch) // MUTABLE -- set to true for WATCH target as opposed to normal TEE { -#ifdef ZT_RULES_ENGINE_DEBUGGING - char dpbuf[1024]; // used by FILTER_TRACE macro - std::vector dlog; -#endif // ZT_RULES_ENGINE_DEBUGGING - // Set to true if we are a TEE/REDIRECT/WATCH target bool superAccept = false; @@ -193,6 +113,8 @@ static _doZtFilterResult _doZtFilter( // ACTION with no MATCH entries preceding it is always taken. uint8_t thisSetMatches = 1; + rrl.clear(); + for(unsigned int rn=0;rnidentity.address()) { if (inbound) { -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"interpreted as super-ACCEPT on inbound since we are target"); -#endif // ZT_RULES_ENGINE_DEBUGGING return DOZTFILTER_SUPER_ACCEPT; } else { -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"skipped as no-op on outbound since we are target"); - dlog.clear(); -#endif // ZT_RULES_ENGINE_DEBUGGING } } else if (fwdAddr == ztDest) { -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,"skipped as no-op because destination is already target"); - dlog.clear(); -#endif // ZT_RULES_ENGINE_DEBUGGING } else { if (rt == ZT_NETWORK_RULE_ACTION_REDIRECT) { -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_REDIRECT",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); -#endif // ZT_RULES_ENGINE_DEBUGGING ztDest = fwdAddr; return DOZTFILTER_REDIRECT; } else { -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); - dlog.clear(); -#endif // ZT_RULES_ENGINE_DEBUGGING cc = fwdAddr; ccLength = (rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen; ccWatch = (rt == ZT_NETWORK_RULE_ACTION_WATCH); @@ -259,18 +154,10 @@ static _doZtFilterResult _doZtFilter( } continue; case ZT_NETWORK_RULE_ACTION_BREAK: -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace("ACTION_BREAK",thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); - dlog.clear(); -#endif // ZT_RULES_ENGINE_DEBUGGING return DOZTFILTER_NO_MATCH; // Unrecognized ACTIONs are ignored as no-ops default: -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); - dlog.clear(); -#endif // ZT_RULES_ENGINE_DEBUGGING continue; } } else { @@ -290,10 +177,6 @@ static _doZtFilterResult _doZtFilter( } } -#ifdef ZT_RULES_ENGINE_DEBUGGING - _dumpFilterTrace(_rtn(rt),thisSetMatches,inbound,ztSource,ztDest,macSource,macDest,dlog,frameLen,etherType,(const char *)0); - dlog.clear(); -#endif // ZT_RULES_ENGINE_DEBUGGING thisSetMatches = 1; // reset to default true for next batch of entries continue; } @@ -301,8 +184,10 @@ static _doZtFilterResult _doZtFilter( // Circuit breaker: no need to evaluate an AND if the set's match state // is currently false since anything AND false is false. - if ((!thisSetMatches)&&(!(rules[rn].t & 0x40))) + if ((!thisSetMatches)&&(!(rules[rn].t & 0x40))) { + rrl.logSkipped(rn,thisSetMatches); continue; + } // If this was not an ACTION evaluate next MATCH and update thisSetMatches with (AND [result]) uint8_t thisRuleMatches = 0; @@ -310,106 +195,82 @@ static _doZtFilterResult _doZtFilter( switch(rt) { case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt()); - FILTER_TRACE("%u %s %c %.10llx==%.10llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.zt,ztSource.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztDest.toInt()); - FILTER_TRACE("%u %s %c %.10llx==%.10llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.zt,ztDest.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_ID: thisRuleMatches = (uint8_t)(rules[rn].v.vlanId == (uint16_t)vlanId); - FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanId,(unsigned int)vlanId,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_PCP: // NOT SUPPORTED YET thisRuleMatches = (uint8_t)(rules[rn].v.vlanPcp == 0); - FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanPcp,0,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_VLAN_DEI: // NOT SUPPORTED YET thisRuleMatches = (uint8_t)(rules[rn].v.vlanDei == 0); - FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.vlanDei,0,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macSource); - FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macSource.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: thisRuleMatches = (uint8_t)(MAC(rules[rn].v.mac,6) == macDest); - FILTER_TRACE("%u %s %c %.12llx=%.12llx -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),rules[rn].v.mac,macDest.toInt(),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0))); - FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str(),InetAddress((const void *)(frameData + 12),4,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IPv4] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV4_DEST: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0))); - FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).toString().c_str(),InetAddress((const void *)(frameData + 16),4,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IPv4] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0))); - FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str(),InetAddress((const void *)(frameData + 8),16,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IPV6_DEST: if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0))); - FILTER_TRACE("%u %s %c %s contains %s -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).toString().c_str(),InetAddress((const void *)(frameData + 24),16,0).toIpString().c_str(),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_TOS: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { - //thisRuleMatches = (uint8_t)(rules[rn].v.ipTos == ((frameData[1] & 0xfc) >> 2)); const uint8_t tosMasked = frameData[1] & rules[rn].v.ipTos.mask; thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); - FILTER_TRACE("%u %s %c (IPv4) %u&%u==%u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)tosMasked,(unsigned int)rules[rn].v.ipTos.mask,(unsigned int)rules[rn].v.ipTos.value[0],(unsigned int)rules[rn].v.ipTos.value[1],(unsigned int)thisRuleMatches); } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) { const uint8_t tosMasked = (((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f)) & rules[rn].v.ipTos.mask; thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1])); - FILTER_TRACE("%u %s %c (IPv4) %u&%u==%u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)tosMasked,(unsigned int)rules[rn].v.ipTos.mask,(unsigned int)rules[rn].v.ipTos.value[0],(unsigned int)rules[rn].v.ipTos.value[1],(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == frameData[9]); - FILTER_TRACE("%u %s %c (IPv4) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipProtocol,(unsigned int)frameData[9],(unsigned int)thisRuleMatches); } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto); - FILTER_TRACE("%u %s %c (IPv6) %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.ipProtocol,proto,(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_ETHERTYPE: thisRuleMatches = (uint8_t)(rules[rn].v.etherType == (uint16_t)etherType); - FILTER_TRACE("%u %s %c %u==%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.etherType,etherType,(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_ICMP: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { @@ -425,14 +286,11 @@ static _doZtFilterResult _doZtFilter( } else { thisRuleMatches = 0; } - FILTER_TRACE("%u %s %c (IPv4) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[ihl],(int)rules[rn].v.icmp.type,(int)frameData[ihl+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [IPv4 frame invalid] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not ICMP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; @@ -447,21 +305,16 @@ static _doZtFilterResult _doZtFilter( } else { thisRuleMatches = 0; } - FILTER_TRACE("%u %s %c (IPv6) icmp-type:%d==%d icmp-code:%d==%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(int)frameData[pos],(int)rules[rn].v.icmp.type,(int)frameData[pos+1],(((rules[rn].v.icmp.flags & 0x01) != 0) ? (int)rules[rn].v.icmp.code : -1),(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not ICMPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; - break; case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) { @@ -482,7 +335,6 @@ static _doZtFilterResult _doZtFilter( } thisRuleMatches = (p >= 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; - FILTER_TRACE("%u %s %c (IPv4) %d in %d-%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),p,(int)rules[rn].v.port[0],(int)rules[rn].v.port[1],(unsigned int)thisRuleMatches); } else if (etherType == ZT_ETHERTYPE_IPV6) { unsigned int pos = 0,proto = 0; if (_ipv6GetPayload(frameData,frameLen,pos,proto)) { @@ -501,14 +353,11 @@ static _doZtFilterResult _doZtFilter( break; } thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0; - FILTER_TRACE("%u %s %c (IPv6) %d in %d-%d -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),p,(int)rules[rn].v.port[0],(int)rules[rn].v.port[1],(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [invalid IPv6] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c [frame not IP] -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } break; case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: { @@ -570,15 +419,12 @@ static _doZtFilterResult _doZtFilter( } } thisRuleMatches = (uint8_t)((cf & rules[rn].v.characteristics) != 0); - FILTER_TRACE("%u %s %c (%.16llx | %.16llx)!=0 -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),cf,rules[rn].v.characteristics,(unsigned int)thisRuleMatches); } break; case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1])); - FILTER_TRACE("%u %s %c %u in %u-%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),frameLen,(unsigned int)rules[rn].v.frameSize[0],(unsigned int)rules[rn].v.frameSize[1],(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_RANDOM: thisRuleMatches = (uint8_t)((uint32_t)(RR->node->prng() & 0xffffffffULL) <= rules[rn].v.randomProbability); - FILTER_TRACE("%u %s %c -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)thisRuleMatches); break; case ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE: case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: @@ -594,26 +440,20 @@ static _doZtFilterResult _doZtFilter( if (rt == ZT_NETWORK_RULE_MATCH_TAGS_DIFFERENCE) { const uint32_t diff = (ltv > rtv) ? (ltv - rtv) : (rtv - ltv); thisRuleMatches = (uint8_t)(diff <= rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%u remote:%u difference:%u<=%u -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,diff,(unsigned int)rules[rn].v.tag.value,thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND) { thisRuleMatches = (uint8_t)((ltv & rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x & remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR) { thisRuleMatches = (uint8_t)((ltv | rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x | remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR) { thisRuleMatches = (uint8_t)((ltv ^ rtv) == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u local:%.8x ^ remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_EQUAL) { thisRuleMatches = (uint8_t)((ltv == rules[rn].v.tag.value)&&(rtv == rules[rn].v.tag.value)); - FILTER_TRACE("%u %s %c TAG %u local:%.8x and remote:%.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,ltv,rtv,(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { // sanity check, can't really happen thisRuleMatches = 0; } } else { if ((inbound)&&(!superAccept)) { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c remote tag %u not found -> 0 (inbound side is strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { // Outbound side is not strict since if we have to match both tags and // we are sending a first packet to a recipient, we probably do not know @@ -621,43 +461,35 @@ static _doZtFilterResult _doZtFilter( // once we get their tag. If we are a tee/redirect target we are also // not strict since we likely do not have these tags. thisRuleMatches = 1; - FILTER_TRACE("%u %s %c remote tag %u not found -> 1 (outbound side and TEE/REDIRECT targets are not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } break; case ZT_NETWORK_RULE_MATCH_TAG_SENDER: case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: { if (superAccept) { thisRuleMatches = 1; - FILTER_TRACE("%u %s %c we are a TEE/REDIRECT target -> 1",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '=')); } else if ( ((rt == ZT_NETWORK_RULE_MATCH_TAG_SENDER)&&(inbound)) || ((rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER)&&(!inbound)) ) { const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0); if (remoteTag) { thisRuleMatches = (uint8_t)(remoteTag->value() == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,remoteTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { if (rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER) { // If we are checking the receiver and this is an outbound packet, we // can't be strict since we may not yet know the receiver's tag. thisRuleMatches = 1; - FILTER_TRACE("%u %s %c (inbound) remote tag %u not found -> 1 (outbound receiver match is not strict)",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c (inbound) remote tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } } else { // sender and outbound or receiver and inbound const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate()); if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) { thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value); - FILTER_TRACE("%u %s %c TAG %u %.8x == %.8x -> %u",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id,localTag->value(),(unsigned int)rules[rn].v.tag.value,(unsigned int)thisRuleMatches); } else { thisRuleMatches = 0; - FILTER_TRACE("%u %s %c local tag %u not found -> 0",rn,_rtn(rt),(((rules[rn].t & 0x80) != 0) ? '!' : '='),(unsigned int)rules[rn].v.tag.id); } } } break; @@ -669,6 +501,8 @@ static _doZtFilterResult _doZtFilter( break; } + rrl.log(rn,thisRuleMatches,thisSetMatches); + if ((rules[rn].t & 0x40)) thisSetMatches |= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); else thisSetMatches &= (thisRuleMatches ^ ((rules[rn].t >> 7) & 1)); @@ -761,33 +595,34 @@ bool Network::filterOutgoingPacket( const uint64_t now = RR->node->now(); Address ztFinalDest(ztDest); int localCapabilityIndex = -1; - bool accept = false; + int accept = 0; + Trace::RuleResultLog rrl,crrl; + Address cc; + unsigned int ccLength = 0; + bool ccWatch = false; Mutex::Lock _l(_lock); Membership *const membership = (ztDest) ? _memberships.get(ztDest) : (Membership *)0; - Address cc; - unsigned int ccLength = 0; - bool ccWatch = false; - switch(_doZtFilter(RR,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { + switch(_doZtFilter(RR,rrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { - case DOZTFILTER_NO_MATCH: + case DOZTFILTER_NO_MATCH: { for(unsigned int c=0;c<_config.capabilityCount;++c) { ztFinalDest = ztDest; // sanity check, shouldn't be possible if there was no match Address cc2; unsigned int ccLength2 = 0; bool ccWatch2 = false; - switch (_doZtFilter(RR,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2)) { + switch (_doZtFilter(RR,crrl,_config,membership,false,ztSource,ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.capabilities[c].rules(),_config.capabilities[c].ruleCount(),cc2,ccLength2,ccWatch2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: - case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side + case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side in capabilities localCapabilityIndex = (int)c; - accept = true; + accept = 1; if ((!noTee)&&(cc2)) { Membership &m2 = _membership(cc2); @@ -809,15 +644,20 @@ bool Network::filterOutgoingPacket( if (accept) break; } - break; + } break; case DOZTFILTER_DROP: + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(Trace::RuleResultLog *)0,(Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); return false; case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() case DOZTFILTER_ACCEPT: - case DOZTFILTER_SUPER_ACCEPT: // no difference in behavior on outbound side - accept = true; + accept = 1; + break; + + case DOZTFILTER_SUPER_ACCEPT: + accept = 2; break; } @@ -854,11 +694,17 @@ bool Network::filterOutgoingPacket( outp.compress(); RR->sw->send(tPtr,outp,true); + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); return false; // DROP locally, since we redirected } else { + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,1); return true; } } else { + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); return false; } } @@ -875,26 +721,27 @@ int Network::filterIncomingPacket( const unsigned int vlanId) { Address ztFinalDest(ztDest); + Trace::RuleResultLog rrl,crrl; int accept = 0; + Address cc; + unsigned int ccLength = 0; + bool ccWatch = false; + const Capability *c = (Capability *)0; Mutex::Lock _l(_lock); Membership &membership = _membership(sourcePeer->address()); - Address cc; - unsigned int ccLength = 0; - bool ccWatch = false; - switch (_doZtFilter(RR,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { + switch (_doZtFilter(RR,rrl,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,cc,ccLength,ccWatch)) { case DOZTFILTER_NO_MATCH: { Membership::CapabilityIterator mci(membership,_config); - const Capability *c; while ((c = mci.next())) { ztFinalDest = ztDest; // sanity check, should be unmodified if there was no match Address cc2; unsigned int ccLength2 = 0; bool ccWatch2 = false; - switch(_doZtFilter(RR,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),cc2,ccLength2,ccWatch2)) { + switch(_doZtFilter(RR,crrl,_config,&membership,true,sourcePeer->address(),ztFinalDest,macSource,macDest,frameData,frameLen,etherType,vlanId,c->rules(),c->ruleCount(),cc2,ccLength2,ccWatch2)) { case DOZTFILTER_NO_MATCH: case DOZTFILTER_DROP: // explicit DROP in a capability just terminates its evaluation and is an anti-pattern break; @@ -927,6 +774,8 @@ int Network::filterIncomingPacket( } break; case DOZTFILTER_DROP: + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(Trace::RuleResultLog *)0,(Capability *)0,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,0); return 0; // DROP case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() @@ -966,10 +815,14 @@ int Network::filterIncomingPacket( outp.compress(); RR->sw->send(tPtr,outp,true); + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(c) ? &crrl : (Trace::RuleResultLog *)0,c,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,0); return 0; // DROP locally, since we redirected } } + if (_config.remoteTraceTarget) + RR->t->networkFilter(*this,rrl,(c) ? &crrl : (Trace::RuleResultLog *)0,c,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,accept); return accept; } @@ -1025,15 +878,10 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add totalLength = chunk.at(ptr); ptr += 4; chunkIndex = chunk.at(ptr); ptr += 4; - if (((chunkIndex + chunkLen) > totalLength)||(totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY)) { // >= since we need room for a null at the end - TRACE("discarded chunk from %s: invalid length or length overflow",source.toString().c_str()); + if (((chunkIndex + chunkLen) > totalLength)||(totalLength >= ZT_NETWORKCONFIG_DICT_CAPACITY)) // >= since we need room for a null at the end return 0; - } - - if ((chunk[ptr] != 1)||(chunk.at(ptr + 1) != ZT_C25519_SIGNATURE_LEN)) { - TRACE("discarded chunk from %s: unrecognized signature type",source.toString().c_str()); + if ((chunk[ptr] != 1)||(chunk.at(ptr + 1) != ZT_C25519_SIGNATURE_LEN)) return 0; - } const uint8_t *sig = reinterpret_cast(chunk.field(ptr + 3,ZT_C25519_SIGNATURE_LEN)); // We can use the signature, which is unique per chunk, to get a per-chunk ID for local deduplication use @@ -1058,14 +906,10 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add // If it's not a duplicate, check chunk signature const Identity controllerId(RR->topology->getIdentity(tPtr,controller())); - if (!controllerId) { // we should always have the controller identity by now, otherwise how would we have queried it the first time? - TRACE("unable to verify chunk from %s: don't have controller identity",source.toString().c_str()); + if (!controllerId) // we should always have the controller identity by now, otherwise how would we have queried it the first time? return 0; - } - if (!controllerId.verify(chunk.field(start,ptr - start),ptr - start,sig,ZT_C25519_SIGNATURE_LEN)) { - TRACE("discarded chunk from %s: signature check failed",source.toString().c_str()); + if (!controllerId.verify(chunk.field(start,ptr - start),ptr - start,sig,ZT_C25519_SIGNATURE_LEN)) return 0; - } // New properly verified chunks can be flooded "virally" through the network if (fastPropagate) { @@ -1095,7 +939,7 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add c = &(_incomingConfigChunks[i]); } } else { - TRACE("discarded single-chunk unsigned legacy config: this is only allowed if the sender is the controller itself"); + // Single-chunk unsigned legacy configs are only allowed from the controller itself return 0; } @@ -1188,9 +1032,7 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD } return 2; // OK and configuration has changed - } catch ( ... ) { - TRACE("ignored invalid configuration for network %.16llx",(unsigned long long)_id); - } + } catch ( ... ) {} // ignore invalid configs return 0; } @@ -1293,6 +1135,8 @@ void Network::requestConfiguration(void *tPtr) rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); + RR->t->networkConfigRequestSent(*this,ctrl); + if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { RR->localNetworkController->request(_id,InetAddress(),0xffffffffffffffffULL,RR->identity,rmd); @@ -1302,8 +1146,6 @@ void Network::requestConfiguration(void *tPtr) return; } - TRACE("requesting netconf for network %.16llx from controller %s",(unsigned long long)_id,ctrl.toString().c_str()); - Packet outp(ctrl,RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REQUEST); outp.append((uint64_t)_id); const unsigned int rmdSize = rmd.sizeBytes(); @@ -1337,9 +1179,7 @@ bool Network::gate(void *tPtr,const SharedPtr &peer) return true; } } - } catch ( ... ) { - TRACE("gate() check failed for peer %s: unexpected exception",peer->address().toString().c_str()); - } + } catch ( ... ) {} return false; } diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index e5929923..0bf4bc19 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -47,6 +47,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false; @@ -217,6 +218,7 @@ bool NetworkConfig::fromDictionary(const DictionaryremoteTraceTarget = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET); this->multicastLimit = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,0); d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name,sizeof(this->name)); diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index fdd078d5..8b3b3619 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -159,6 +159,8 @@ namespace ZeroTier { #define ZT_NETWORKCONFIG_DICT_KEY_REVISION "r" // address of member #define ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO "id" +// remote trace target +#define ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET "tt" // flags(hex) #define ZT_NETWORKCONFIG_DICT_KEY_FLAGS "f" // integer(hex) @@ -462,6 +464,11 @@ public: */ Address issuedTo; + /** + * If non-NULL, remote traces related to this network are sent here + */ + Address remoteTraceTarget; + /** * Flags (64-bit) */ diff --git a/node/Node.cpp b/node/Node.cpp index e28accee..c54ca450 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -46,6 +46,7 @@ #include "Identity.hpp" #include "SelfAwareness.hpp" #include "Network.hpp" +#include "Trace.hpp" namespace ZeroTier { @@ -108,6 +109,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 } try { + RR->t = new Trace(RR); RR->sw = new Switch(RR); RR->mc = new Multicaster(RR); RR->topology = new Topology(RR,tptr); @@ -133,6 +135,7 @@ Node::~Node() delete RR->topology; delete RR->mc; delete RR->sw; + delete RR->t; } ZT_ResultCode Node::processWirePacket( diff --git a/node/Node.hpp b/node/Node.hpp index 40903f7c..57b99fe9 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -48,13 +48,6 @@ #include "NetworkController.hpp" #include "Hashtable.hpp" -#undef TRACE -#ifdef ZT_TRACE -#define TRACE(f,...) RR->node->postTrace(__FILE__,__LINE__,f,##__VA_ARGS__) -#else -#define TRACE(f,...) {} -#endif - // Bit mask for "expecting reply" hash #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 #define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 diff --git a/node/OutboundMulticast.cpp b/node/OutboundMulticast.cpp index a2341ffd..04ba2c2a 100644 --- a/node/OutboundMulticast.cpp +++ b/node/OutboundMulticast.cpp @@ -65,18 +65,6 @@ void OutboundMulticast::init( if (gatherLimit) flags |= 0x02; - /* - TRACE(">>MC %.16llx INIT %.16llx/%s limit %u gatherLimit %u from %s to %s length %u", - (unsigned long long)this, - nwid, - dest.toString().c_str(), - limit, - gatherLimit, - (src) ? src.toString().c_str() : MAC(RR->identity.address(),nwid).toString().c_str(), - dest.toString().c_str(), - len); - */ - _packet.setSource(RR->identity.address()); _packet.setVerb(Packet::VERB_MULTICAST_FRAME); _packet.append((uint64_t)nwid); @@ -98,7 +86,6 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,void *tPtr,const A const SharedPtr nw(RR->node->network(_nwid)); const Address toAddr2(toAddr); if ((nw)&&(nw->filterOutgoingPacket(tPtr,true,RR->identity.address(),toAddr2,_macSrc,_macDest,_frameData,_frameLen,_etherType,0))) { - //TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str()); _packet.newInitializationVector(); _packet.setDestination(toAddr2); RR->node->expectReplyTo(_packet.packetId()); diff --git a/node/Packet.cpp b/node/Packet.cpp index 6e1b36ac..d3f7dfd6 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -1061,50 +1061,6 @@ static inline int LZ4_decompress_safe(const char* source, char* dest, int compre const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; -#ifdef ZT_TRACE - -const char *Packet::verbString(Verb v) -{ - switch(v) { - case VERB_NOP: return "NOP"; - case VERB_HELLO: return "HELLO"; - case VERB_ERROR: return "ERROR"; - case VERB_OK: return "OK"; - case VERB_WHOIS: return "WHOIS"; - case VERB_RENDEZVOUS: return "RENDEZVOUS"; - case VERB_FRAME: return "FRAME"; - case VERB_EXT_FRAME: return "EXT_FRAME"; - case VERB_ECHO: return "ECHO"; - case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; - case VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; - case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; - case VERB_NETWORK_CONFIG: return "NETWORK_CONFIG"; - case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; - case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; - case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; - case VERB_USER_MESSAGE: return "USER_MESSAGE"; - } - return "(unknown)"; -} - -const char *Packet::errorString(ErrorCode e) -{ - switch(e) { - case ERROR_NONE: return "NONE"; - case ERROR_INVALID_REQUEST: return "INVALID_REQUEST"; - case ERROR_BAD_PROTOCOL_VERSION: return "BAD_PROTOCOL_VERSION"; - case ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND"; - case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION"; - case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION"; - case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE"; - case ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED"; - case ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST"; - } - return "(unknown)"; -} - -#endif // ZT_TRACE - void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) { uint8_t mangledKey[32]; diff --git a/node/Packet.hpp b/node/Packet.hpp index a76d4180..4941e96a 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -42,12 +42,6 @@ #include "Utils.hpp" #include "Buffer.hpp" -//#ifdef ZT_USE_SYSTEM_LZ4 -//#include -//#else -//#include "../ext/lz4/lz4.h" -//#endif - /** * Protocol version -- incremented only for major changes * @@ -969,7 +963,27 @@ public: * ZeroTier, Inc. itself. We recommend making up random ones for your own * implementations. */ - VERB_USER_MESSAGE = 0x14 + VERB_USER_MESSAGE = 0x14, + + /** + * A trace for remote debugging or diagnostics: + * <[8] 64-bit instance ID> + * <[2] 16-bit length of Dictionary> + * <[...] dictionary containing trace information> + * + * This message contains a remote trace event. Remote trace events can + * be sent to observers configured at the network level for those that + * pertain directly to actiity on a network, or to global observers if + * locally configured. + * + * The instance ID is a random 64-bit value generated by each ZeroTier + * node on startup. This is helpful in identifying traces from different + * members of a cluster. + * + * The Dictionary serialization format is the same as used for network + * configurations. The maximum size of a trace is 10000 bytes. + */ + VERB_REMOTE_TRACE = 0x15 }; /** @@ -1005,11 +1019,6 @@ public: ERROR_UNWANTED_MULTICAST = 0x08 }; -#ifdef ZT_TRACE - static const char *verbString(Verb v); - static const char *errorString(ErrorCode e); -#endif - template Packet(const Buffer &b) : Buffer(b) diff --git a/node/Peer.cpp b/node/Peer.cpp index e16540b3..79a4bc90 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -33,6 +33,7 @@ #include "Network.hpp" #include "SelfAwareness.hpp" #include "Packet.hpp" +#include "Trace.hpp" namespace ZeroTier { @@ -168,22 +169,25 @@ void Peer::received( if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address())) ) { Mutex::Lock _l(_paths_m); - _PeerPath *potentialNewPeerPath = (_PeerPath *)0; + + _PeerPath *replacablePath = (_PeerPath *)0; if (path->address().ss_family == AF_INET) { if ( ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) && ( (now - _v4Path.sticky) > ZT_PEER_PATH_EXPIRATION ) ) { - potentialNewPeerPath = &_v4Path; + replacablePath = &_v4Path; } } else if (path->address().ss_family == AF_INET6) { if ( ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) && ( (now - _v6Path.sticky) > ZT_PEER_PATH_EXPIRATION ) ) { - potentialNewPeerPath = &_v6Path; + replacablePath = &_v6Path; } } - if (potentialNewPeerPath) { + + if (replacablePath) { if (verb == Packet::VERB_OK) { - potentialNewPeerPath->lr = now; - potentialNewPeerPath->p = path; + RR->t->peerLearnedNewPath(*this,replacablePath->p,path,packetId); + replacablePath->lr = now; + replacablePath->p = path; } else { - TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); + RR->t->peerConfirmingUnknownPath(*this,path,packetId,verb); attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); path->sent(now); } @@ -211,16 +215,6 @@ void Peer::received( } if (pathsToPush.size() > 0) { -#ifdef ZT_TRACE - std::string ps; - for(std::vector::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) { - if (ps.length() > 0) - ps.push_back(','); - ps.append(p->toString()); - } - TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str()); -#endif - std::vector::const_iterator p(pathsToPush.begin()); while (p != pathsToPush.end()) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS); @@ -424,16 +418,27 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now) { - Mutex::Lock _l(_paths_m); - SharedPtr p(RR->topology->getPath(localSocket,remoteAddress)); - attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,p->nextOutgoingCounter()); - if (remoteAddress.ss_family == AF_INET) { - _v4Path.p = p; - _v4Path.sticky = now; - } else if (remoteAddress.ss_family == AF_INET6) { - _v6Path.p = p; - _v6Path.sticky = now; + if ((remoteAddress.ss_family != AF_INET)&&(remoteAddress.ss_family != AF_INET6)) // sanity check + return; + + SharedPtr op; + SharedPtr np(RR->topology->getPath(localSocket,remoteAddress)); + attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,np->nextOutgoingCounter()); + + { + Mutex::Lock _l(_paths_m); + if (remoteAddress.ss_family == AF_INET) { + op = _v4Path.p; + _v4Path.p = np; + _v4Path.sticky = now; + } else if (remoteAddress.ss_family == AF_INET6) { + op = _v6Path.p; + _v6Path.p = np; + _v6Path.sticky = now; + } } + + RR->t->peerRedirected(*this,op,np); } } // namespace ZeroTier diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 94b96d34..0bb78599 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -42,7 +42,7 @@ class Node; class Multicaster; class NetworkController; class SelfAwareness; -class Cluster; +class Trace; /** * Holds global state for an instance of ZeroTier::Node @@ -93,6 +93,7 @@ public: * These are constant and never null after startup unless indicated. */ + Trace *t; Switch *sw; Multicaster *mc; Topology *topology; diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 3e3397f5..173230fb 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -39,6 +39,7 @@ #include "Packet.hpp" #include "Peer.hpp" #include "Switch.hpp" +#include "Trace.hpp" // Entry timeout -- make it fairly long since this is just to prevent stale buildup #define ZT_SELFAWARENESS_ENTRY_TIMEOUT 600000 @@ -81,7 +82,7 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receive if ( (trusted) && ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) { // Changes to external surface reported by trusted peers causes path reset in this scope - TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str()); + RR->t->resettingPathsInScope(reporter,reporterPhysicalAddress,myPhysicalAddress,scope); entry.mySurface = myPhysicalAddress; entry.ts = now; diff --git a/node/Switch.cpp b/node/Switch.cpp index a77ca89e..2fbd243b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -43,26 +43,10 @@ #include "Peer.hpp" #include "SelfAwareness.hpp" #include "Packet.hpp" +#include "Trace.hpp" namespace ZeroTier { -#ifdef ZT_TRACE -static const char *etherTypeName(const unsigned int etherType) -{ - switch(etherType) { - case ZT_ETHERTYPE_IPV4: return "IPV4"; - case ZT_ETHERTYPE_ARP: return "ARP"; - case ZT_ETHERTYPE_RARP: return "RARP"; - case ZT_ETHERTYPE_ATALK: return "ATALK"; - case ZT_ETHERTYPE_AARP: return "AARP"; - case ZT_ETHERTYPE_IPX_A: return "IPX_A"; - case ZT_ETHERTYPE_IPX_B: return "IPX_B"; - case ZT_ETHERTYPE_IPV6: return "IPV6"; - } - return "UNKNOWN"; -} -#endif // ZT_TRACE - Switch::Switch(const RuntimeEnvironment *renv) : RR(renv), _lastBeaconResponse(0), @@ -123,8 +107,6 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre if (relayTo) relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,true); } - } else { - TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str()); } } else { // Fragment looks like ours @@ -143,7 +125,6 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre if ((!rq->timestamp)||(rq->packetId != fragmentPacketId)) { // No packet found, so we received a fragment without its head. - //TRACE("fragment (%u/%u) of %.16llx from %s",fragmentNumber + 1,totalFragments,fragmentPacketId,fromAddr.toString().c_str()); rq->timestamp = now; rq->packetId = fragmentPacketId; @@ -153,14 +134,12 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre rq->complete = false; } else if (!(rq->haveFragments & (1 << fragmentNumber))) { // We have other fragments and maybe the head, so add this one and check - //TRACE("fragment (%u/%u) of %.16llx from %s",fragmentNumber + 1,totalFragments,fragmentPacketId,fromAddr.toString().c_str()); rq->frags[fragmentNumber - 1] = fragment; rq->totalFragments = totalFragments; if (Utils::countBits(rq->haveFragments |= (1 << fragmentNumber)) == totalFragments) { // We have all fragments -- assemble and process full Packet - //TRACE("packet %.16llx is complete, assembling and processing...",fragmentPacketId); for(unsigned int f=1;ffrag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength()); @@ -182,8 +161,6 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre const Address destination(reinterpret_cast(data) + 8,ZT_ADDRESS_LENGTH); const Address source(reinterpret_cast(data) + 13,ZT_ADDRESS_LENGTH); - //TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size()); - if (source == RR->identity.address()) return; @@ -258,8 +235,6 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre if (relayTo) relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true); } - } else { - TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet.source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str()); } } else if ((reinterpret_cast(data)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED) != 0) { // Packet is the head of a fragmented packet series @@ -280,7 +255,6 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre if ((!rq->timestamp)||(rq->packetId != packetId)) { // If we have no other fragments yet, create an entry and save the head - //TRACE("fragment (0/?) of %.16llx from %s",pid,fromAddr.toString().c_str()); rq->timestamp = now; rq->packetId = packetId; @@ -293,7 +267,6 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre if ((rq->totalFragments > 1)&&(Utils::countBits(rq->haveFragments |= 1) == rq->totalFragments)) { // We have all fragments -- assemble and process full Packet - //TRACE("packet %.16llx is complete, assembling and processing...",pid); rq->frag0.init(data,len,path,now); for(unsigned int f=1;ftotalFragments;++f) @@ -333,11 +306,7 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre // -------------------------------------------------------------------- } } - } catch (std::exception &ex) { - TRACE("dropped packet from %s: unexpected exception: %s",fromAddr.toString().c_str(),ex.what()); - } catch ( ... ) { - TRACE("dropped packet from %s: unexpected exception: (unknown)",fromAddr.toString().c_str()); - } + } catch ( ... ) {} // sanity check, should be caught elsewhere } void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len) @@ -349,7 +318,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const bool fromBridged; if ((fromBridged = (from != network->mac()))) { if (!network->config().permitsBridging(RR->identity.address())) { - TRACE("%.16llx: %s -> %s %s not forwarded, bridging disabled or this peer not a bridge",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"not a bridge"); return; } } @@ -371,7 +340,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const multicastGroup = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(((const unsigned char *)data) + 24,4,0)); } else if (!network->config().enableBroadcast()) { // Don't transmit broadcasts if this network doesn't want them - TRACE("%.16llx: dropped broadcast since ff:ff:ff:ff:ff:ff is not enabled",network->id()); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"broadcast disabled"); return; } } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(len >= (40 + 8 + 16))) { @@ -424,7 +393,6 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const if ((v6EmbeddedAddress)&&(v6EmbeddedAddress != RR->identity.address())) { const MAC peerMac(v6EmbeddedAddress,network->id()); - TRACE("IPv6 NDP emulation: %.16llx: forging response for %s/%s",network->id(),v6EmbeddedAddress.toString().c_str(),peerMac.toString().c_str()); uint8_t adv[72]; adv[0] = 0x60; adv[1] = 0x00; adv[2] = 0x00; adv[3] = 0x00; @@ -460,7 +428,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const // Check this after NDP emulation, since that has to be allowed in exactly this case if (network->config().multicastLimit == 0) { - TRACE("%.16llx: dropped multicast: not allowed on network",network->id()); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"multicast disabled"); return; } @@ -471,11 +439,9 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const if (fromBridged) network->learnBridgedMulticastGroup(tPtr,multicastGroup,RR->node->now()); - //TRACE("%.16llx: MULTICAST %s -> %s %s %u",network->id(),from.toString().c_str(),multicastGroup.toString().c_str(),etherTypeName(etherType),len); - // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { - TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -501,7 +467,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const SharedPtr toPeer(RR->topology->getPeer(tPtr,toZT)); if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { - TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -526,7 +492,6 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const send(tPtr,outp,true); } - //TRACE("%.16llx: UNICAST: %s -> %s etherType==%s(%.4x) vlanId==%u len==%u fromBridged==%d includeCom==%d",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),etherType,vlanId,len,(int)fromBridged,(int)includeCom); } else { // Destination is bridged behind a remote peer @@ -534,7 +499,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const // for each ZT destination are also done below. This is the same rationale // and design as for multicast. if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { - TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -583,7 +548,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const outp.compress(); send(tPtr,outp,true); } else { - TRACE("%.16llx: %s -> %s %s packet not sent: filterOutgoingPacket() returned false",network->id(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType)); + RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked (bridge replication)"); } } } @@ -591,11 +556,8 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const void Switch::send(void *tPtr,Packet &packet,bool encrypt) { - if (packet.destination() == RR->identity.address()) { - TRACE("BUG: caught attempt to send() to self, ignored"); + if (packet.destination() == RR->identity.address()) return; - } - if (!_trySend(tPtr,packet,encrypt)) { Mutex::Lock _l(_txQueue_m); _txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt)); @@ -604,13 +566,8 @@ void Switch::send(void *tPtr,Packet &packet,bool encrypt) void Switch::requestWhois(void *tPtr,const Address &addr) { -#ifdef ZT_TRACE - if (addr == RR->identity.address()) { - fprintf(stderr,"FATAL BUG: Switch::requestWhois() caught attempt to WHOIS self" ZT_EOL_S); - abort(); - } -#endif - + if (addr == RR->identity.address()) + return; bool inserted = false; { Mutex::Lock _l(_outstandingWhoisRequests_m); @@ -670,12 +627,10 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) const unsigned long since = (unsigned long)(now - r->lastSent); if (since >= ZT_WHOIS_RETRY_DELAY) { if (r->retries >= ZT_MAX_WHOIS_RETRIES) { - TRACE("WHOIS %s timed out",a->toString().c_str()); _outstandingWhoisRequests.erase(*a); } else { r->lastSent = now; r->peersConsulted[r->retries] = _sendWhoisRequest(tPtr,*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); - TRACE("WHOIS %s (retry %u)",a->toString().c_str(),r->retries); ++r->retries; nextDelay = std::min(nextDelay,(unsigned long)ZT_WHOIS_RETRY_DELAY); } @@ -691,7 +646,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) if (_trySend(tPtr,txi->packet,txi->encrypt)) _txQueue.erase(txi++); else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { - TRACE("TX %s -> %s timed out",txi->packet.source().toString().c_str(),txi->packet.destination().toString().c_str()); + RR->t->txTimedOut(txi->dest); _txQueue.erase(txi++); } else ++txi; } diff --git a/node/Topology.cpp b/node/Topology.cpp index e7bbdfae..edca0180 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -90,11 +90,6 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) { -#ifdef ZT_TRACE - if ((!peer)||(peer->address() == RR->identity.address())) - return SharedPtr(); -#endif - SharedPtr np; { Mutex::Lock _l(_peers_m); @@ -103,16 +98,13 @@ SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) hp = peer; np = hp; } - return np; } SharedPtr Topology::getPeer(void *tPtr,const Address &zta) { - if (zta == RR->identity.address()) { - TRACE("BUG: ignored attempt to getPeer() for self, returned NULL"); + if (zta == RR->identity.address()) return SharedPtr(); - } { Mutex::Lock _l(_peers_m); diff --git a/node/Topology.hpp b/node/Topology.hpp index 5f3e2da1..30e58abc 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -330,12 +330,6 @@ public: Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { -#ifdef ZT_TRACE - if (!(*p)) { - fprintf(stderr,"FATAL BUG: eachPeer() caught NULL peer for %s -- peer pointers in Topology should NEVER be NULL" ZT_EOL_S,a->toString().c_str()); - abort(); - } -#endif f(*this,*((const SharedPtr *)p)); } } diff --git a/node/Trace.cpp b/node/Trace.cpp new file mode 100644 index 00000000..6b68cfe7 --- /dev/null +++ b/node/Trace.cpp @@ -0,0 +1,197 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#include "Trace.hpp" +#include "RuntimeEnvironment.hpp" +#include "Switch.hpp" +#include "Node.hpp" +#include "Utils.hpp" + +namespace ZeroTier { + +#ifdef ZT_TRACE +static const char *packetVerbString(Packet::Verb v) +{ + switch(v) { + case Packet::VERB_NOP: return "NOP"; + case Packet::VERB_HELLO: return "HELLO"; + case Packet::Packet::VERB_ERROR: return "ERROR"; + case Packet::VERB_OK: return "OK"; + case Packet::VERB_WHOIS: return "WHOIS"; + case Packet::VERB_RENDEZVOUS: return "RENDEZVOUS"; + case Packet::VERB_FRAME: return "FRAME"; + case Packet::VERB_EXT_FRAME: return "EXT_FRAME"; + case Packet::VERB_ECHO: return "ECHO"; + case Packet::VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; + case Packet::VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; + case Packet::VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; + case Packet::VERB_NETWORK_CONFIG: return "NETWORK_CONFIG"; + case Packet::VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; + case Packet::VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; + case Packet::VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; + case Packet::VERB_USER_MESSAGE: return "USER_MESSAGE"; + case Packet::VERB_REMOTE_TRACE: return "REMOTE_TRACE"; + } + return "(unknown)"; +} + +static const char *packetErrorString(Packet::ErrorCode e) +{ + switch(e) { + case Packet::ERROR_NONE: return "NONE"; + case Packet::ERROR_INVALID_REQUEST: return "INVALID_REQUEST"; + case Packet::ERROR_BAD_PROTOCOL_VERSION: return "BAD_PROTOCOL_VERSION"; + case Packet::ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND"; + case Packet::ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION"; + case Packet::ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION"; + case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE"; + case Packet::ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED"; + case Packet::ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST"; + } + return "(unknown)"; +} +#endif + +void Trace::resettingPathsInScope(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope) +{ +} + +void Trace::txTimedOut(const Address &destination) +{ +} + +void Trace::peerConfirmingUnknownPath(Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb) +{ +} + +void Trace::peerLearnedNewPath(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId) +{ +} + +void Trace::peerRedirected(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath) +{ +} + +void Trace::outgoingFrameDropped(const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason) +{ +} + +void Trace::incomingPacketTrustedPath(const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) +{ +} + +void Trace::incomingPacketMessageAuthenticationFailure(const SharedPtr &path,const uint64_t packetId,const Address &source) +{ +} + +void Trace::incomingPacketInvalid(const SharedPtr &path,const uint64_t packetId,const Address &source,const Packet::Verb verb,const char *reason) +{ +} + +void Trace::incomingPacketDroppedHELLO(const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason) +{ +} + +void Trace::networkAccessDenied(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested) +{ +} + +void Trace::networkFrameDropped(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac) +{ +} + +void Trace::networkConfigRequestSent(const Network &network,const Address &controller) +{ +} + +void Trace::networkFilter( + const Network &network, + const RuleResultLog &primaryRuleSetLog, + const RuleResultLog *const matchingCapabilityRuleSetLog, + const Capability *const matchingCapability, + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *const frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId, + const bool noTee, + const bool inbound, + const int accept) +{ +} + +void Trace::credentialRejected(const CertificateOfMembership &c,const char *reason) +{ +} + +void Trace::credentialRejected(const CertificateOfOwnership &c,const char *reason) +{ +} + +void Trace::credentialRejected(const CertificateOfRepresentation &c,const char *reason) +{ +} + +void Trace::credentialRejected(const Capability &c,const char *reason) +{ +} + +void Trace::credentialRejected(const Tag &c,const char *reason) +{ +} + +void Trace::credentialRejected(const Revocation &c,const char *reason) +{ +} + +void Trace::credentialAccepted(const CertificateOfMembership &c) +{ +} + +void Trace::credentialAccepted(const CertificateOfOwnership &c) +{ +} + +void Trace::credentialAccepted(const CertificateOfRepresentation &c) +{ +} + +void Trace::credentialAccepted(const Capability &c) +{ +} + +void Trace::credentialAccepted(const Tag &c) +{ +} + +void Trace::credentialAccepted(const Revocation &c) +{ +} + +} // namespace ZeroTier diff --git a/node/Trace.hpp b/node/Trace.hpp new file mode 100644 index 00000000..65d1acf1 --- /dev/null +++ b/node/Trace.hpp @@ -0,0 +1,157 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_TRACE_HPP +#define ZT_TRACE_HPP + +#include +#include +#include +#include + +#include "../include/ZeroTierOne.h" + +#include "Constants.hpp" +#include "SharedPtr.hpp" +#include "Packet.hpp" +#include "Credential.hpp" +#include "InetAddress.hpp" + +namespace ZeroTier { + +class RuntimeEnvironment; +class Address; +class Identity; +class Peer; +class Path; +class Network; +class NetworkConfig; +class MAC; +class CertificateOfMembership; +class CertificateOfOwnership; +class CertificateOfRepresentation; +class Revocation; +class Tag; +class Capability; + +/** + * Remote tracing and trace logging handler + */ +class Trace +{ +public: + /** + * Filter rule evaluation result log + * + * Each rule in a rule set gets a four-bit log entry. A log entry + * of zero means not evaluated. Otherwise each four-bit log entry + * contains two two-bit values of 01 for 'false' and 10 for 'true'. + * As with four-bit rules an 00 value here means this was not + * evaluated or was not relevant. + */ + class RuleResultLog + { + public: + RuleResultLog() {} + + inline void log(const unsigned int rn,const uint8_t thisRuleMatches,const uint8_t thisSetMatches) + { + _l[rn >> 1] |= ( ((thisRuleMatches + 1) << 2) | (thisSetMatches + 1) ) << ((rn & 1) << 2); + } + inline void logSkipped(const unsigned int rn,const uint8_t thisSetMatches) + { + _l[rn >> 1] |= (thisSetMatches + 1) << ((rn & 1) << 2); + } + + inline void clear() + { + memset(_l,0,sizeof(_l)); + } + + inline const uint8_t *data() const { return _l; } + inline unsigned int sizeBytes() const { return (unsigned int)sizeof(_l); } + + private: + uint8_t _l[ZT_MAX_NETWORK_RULES / 2]; + }; + + Trace(const RuntimeEnvironment *renv) : RR(renv) {} + + void resettingPathsInScope(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope); + void txTimedOut(const Address &destination); + + void peerConfirmingUnknownPath(Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb); + void peerLearnedNewPath(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId); + void peerRedirected(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath); + + void outgoingFrameDropped(const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason); + + void incomingPacketTrustedPath(const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved); + void incomingPacketMessageAuthenticationFailure(const SharedPtr &path,const uint64_t packetId,const Address &source); + void incomingPacketInvalid(const SharedPtr &path,const uint64_t packetId,const Address &source,const Packet::Verb verb,const char *reason); + void incomingPacketDroppedHELLO(const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason); + + void networkAccessDenied(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested); + void networkFrameDropped(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac); + + void networkConfigRequestSent(const Network &network,const Address &controller); + void networkFilter( + const Network &network, + const RuleResultLog &primaryRuleSetLog, + const RuleResultLog *const matchingCapabilityRuleSetLog, + const Capability *const matchingCapability, + const Address &ztSource, + const Address &ztDest, + const MAC &macSource, + const MAC &macDest, + const uint8_t *const frameData, + const unsigned int frameLen, + const unsigned int etherType, + const unsigned int vlanId, + const bool noTee, + const bool inbound, + const int accept); + + void credentialRejected(const CertificateOfMembership &c,const char *reason); + void credentialRejected(const CertificateOfOwnership &c,const char *reason); + void credentialRejected(const CertificateOfRepresentation &c,const char *reason); + void credentialRejected(const Capability &c,const char *reason); + void credentialRejected(const Tag &c,const char *reason); + void credentialRejected(const Revocation &c,const char *reason); + void credentialAccepted(const CertificateOfMembership &c); + void credentialAccepted(const CertificateOfOwnership &c); + void credentialAccepted(const CertificateOfRepresentation &c); + void credentialAccepted(const Capability &c); + void credentialAccepted(const Tag &c); + void credentialAccepted(const Revocation &c); + +private: + const RuntimeEnvironment *const RR; +}; + +} // namespace ZeroTier + +#endif diff --git a/objects.mk b/objects.mk index 3a8bd645..ed396378 100644 --- a/objects.mk +++ b/objects.mk @@ -23,6 +23,7 @@ CORE_OBJS=\ node/Switch.o \ node/Tag.o \ node/Topology.o \ + node/Trace.o \ node/Utils.o ONE_OBJS=\ -- cgit v1.2.3 From 495c5ce81ddb245e21f21325927236d0f666f6cf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Jul 2017 10:51:05 -0700 Subject: Bunch of remote tracing work. --- node/Dictionary.hpp | 29 ++++ node/Identity.cpp | 2 +- node/IncomingPacket.cpp | 180 ++++++++++++------------ node/Membership.cpp | 22 +-- node/Membership.hpp | 5 + node/Network.cpp | 23 ++-- node/Network.hpp | 13 ++ node/Node.cpp | 7 +- node/Node.hpp | 3 + node/Packet.hpp | 5 +- node/Peer.cpp | 9 +- node/Peer.hpp | 4 +- node/SelfAwareness.cpp | 2 +- node/Switch.cpp | 16 +-- node/Trace.cpp | 353 ++++++++++++++++++++++++++++++++++++++++++++---- node/Trace.hpp | 57 ++++---- 16 files changed, 552 insertions(+), 178 deletions(-) (limited to 'node') diff --git a/node/Dictionary.hpp b/node/Dictionary.hpp index 6cbbfc0e..061dcac1 100644 --- a/node/Dictionary.hpp +++ b/node/Dictionary.hpp @@ -288,6 +288,21 @@ public: return dfl; } + /** + * Get an unsigned int64 stored as hex in the dictionary + * + * @param key Key to look up + * @param dfl Default value or 0 if unspecified + * @return Decoded hex UInt value or 'dfl' if not found + */ + inline int64_t getI(const char *key,int64_t dfl = 0) const + { + char tmp[128]; + if (this->get(key,tmp,sizeof(tmp)) >= 1) + return Utils::hexStrTo64(tmp); + return dfl; + } + /** * Add a new key=value pair * @@ -394,6 +409,20 @@ public: return this->add(key,Utils::hex(value,tmp),-1); } + /** + * Add a 64-bit integer (unsigned) as a hex value + */ + inline bool add(const char *key,int64_t value) + { + char tmp[32]; + if (value >= 0) { + return this->add(key,Utils::hex((uint64_t)value,tmp),-1); + } else { + tmp[0] = '-'; + return this->add(key,Utils::hex((uint64_t)(value * -1),tmp+1),-1); + } + } + /** * Add a 64-bit integer (unsigned) as a hex value */ diff --git a/node/Identity.cpp b/node/Identity.cpp index 3b00b4c0..dba27d1c 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -151,7 +151,7 @@ char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_ Utils::hex(_privateKey->data,ZT_C25519_PRIVATE_KEY_LEN,p); p += ZT_C25519_PRIVATE_KEY_LEN * 2; } - *(p++) = (char)0; + *p = (char)0; return buf; } diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e1fb180c..a5875d1e 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -66,10 +66,10 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) // packets are dropped on the floor. const uint64_t tpid = trustedPathId(); if (RR->topology->shouldInboundPathBeTrusted(_path->address(),tpid)) { - RR->t->incomingPacketTrustedPath(_path,packetId(),sourceAddress,tpid,true); + RR->t->incomingPacketTrustedPath(tPtr,_path,packetId(),sourceAddress,tpid,true); trusted = true; } else { - RR->t->incomingPacketTrustedPath(_path,packetId(),sourceAddress,tpid,false); + RR->t->incomingPacketTrustedPath(tPtr,_path,packetId(),sourceAddress,tpid,false); return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { @@ -82,14 +82,14 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) if (!trusted) { if (!dearmor(peer->key())) { //fprintf(stderr,"dropped packet from %s(%s), MAC authentication failed (size: %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); - RR->t->incomingPacketMessageAuthenticationFailure(_path,packetId(),sourceAddress); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops()); return true; } } if (!uncompress()) { //fprintf(stderr,"dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); - RR->t->incomingPacketInvalid(_path,packetId(),sourceAddress,Packet::VERB_NOP,"LZ4 decompression failed"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),sourceAddress,hops(),Packet::VERB_NOP,"LZ4 decompression failed"); return true; } @@ -97,7 +97,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) switch(v) { //case Packet::VERB_NOP: default: // ignore unknown verbs, but if they pass auth check they are "received" - peer->received(tPtr,_path,hops(),packetId(),v,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),v,0,Packet::VERB_NOP,false,0); return true; case Packet::VERB_HELLO: return _doHELLO(RR,tPtr,true); @@ -122,7 +122,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) return false; } } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),sourceAddress,Packet::VERB_NOP,"unexpected exception in tryDecode() (outer)"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),sourceAddress,hops(),verb(),"unexpected exception in tryDecode() (outer)"); return true; } } @@ -133,6 +133,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID); const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE]; + uint64_t networkId = 0; /* Security note: we do not gate doERROR() with expectingReplyTo() to * avoid having to log every outgoing packet ID. Instead we put the @@ -170,7 +171,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { // Peers can send this in response to frames if they do not have a recent enough COM from us - const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); + const SharedPtr network(RR->node->network(networkId)); const uint64_t now = RR->node->now(); if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) network->pushCredentialsNow(tPtr,peer->address(),now); @@ -186,7 +188,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar case Packet::ERROR_UNWANTED_MULTICAST: { // Members of networks can use this error to indicate that they no longer // want to receive multicasts on a given channel. - const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); + const SharedPtr network(RR->node->network(networkId)); if ((network)&&(network->gate(tPtr,peer))) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); RR->mc->remove(network->id(),mg,peer->address()); @@ -196,9 +199,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar default: break; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,networkId); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_ERROR,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_ERROR,"unexpected exception"); } return true; } @@ -219,11 +222,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); if (protoVersion < ZT_PROTO_VERSION_MIN) { - RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"protocol version too old"); + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"protocol version too old"); return true; } if (fromAddress != id.address()) { - RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"identity/address mismatch"); + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"identity/address mismatch"); return true; } @@ -241,7 +244,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { if (dearmor(key)) { // ensure packet is authentic, otherwise drop - RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"address collision"); + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"address collision"); Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)Packet::VERB_HELLO); outp.append((uint64_t)pid); @@ -249,10 +252,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool outp.armor(key,true,_path->nextOutgoingCounter()); _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } else { - RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); } } else { - RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); } return true; @@ -260,7 +263,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool // Identity is the same as the one we already have -- check packet integrity if (!dearmor(peer->key())) { - RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); return true; } @@ -272,26 +275,26 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool // Sanity check: this basically can't happen if (alreadyAuthenticated) { - RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"illegal alreadyAuthenticated state"); + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"illegal alreadyAuthenticated state"); return true; } // Check rate limits if (!RR->node->rateGateIdentityVerification(now,_path->address())) { - RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"rate limit exceeded"); + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"rate limit exceeded"); return true; } // Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap) SharedPtr newPeer(new Peer(RR,RR->identity,id)); if (!dearmor(newPeer->key())) { - RR->t->incomingPacketMessageAuthenticationFailure(_path,pid,fromAddress); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); return true; } // Check that identity's address is valid as per the derivation function if (!id.locallyValidate()) { - RR->t->incomingPacketDroppedHELLO(_path,pid,fromAddress,"invalid identity"); + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"invalid identity"); return true; } @@ -414,9 +417,9 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool _path->send(RR,tPtr,outp.data(),outp.size(),now); peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false,0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_HELLO,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_HELLO,"unexpected exception"); } return true; } @@ -426,6 +429,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP try { const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); + uint64_t networkId = 0; if (!RR->node->expectingReplyTo(inRePacketId)) return true; @@ -491,27 +495,28 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP break; case Packet::VERB_NETWORK_CONFIG_REQUEST: { - const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_OK_IDX_PAYLOAD))); + networkId = at(ZT_PROTO_VERB_OK_IDX_PAYLOAD); + const SharedPtr network(RR->node->network(networkId)); if (network) network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); } break; case Packet::VERB_MULTICAST_GATHER: { - const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); - const SharedPtr network(RR->node->network(nwid)); + networkId = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(networkId)); if (network) { const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); - RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + RR->mc->addMultiple(tPtr,RR->node->now(),networkId,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); } } break; case Packet::VERB_MULTICAST_FRAME: { const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS]; - const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID); + networkId = at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID); const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI)); - const SharedPtr network(RR->node->network(nwid)); + const SharedPtr network(RR->node->network(networkId)); if (network) { unsigned int offset = 0; @@ -527,7 +532,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; unsigned int totalKnown = at(offset); offset += 4; unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(tPtr,RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown); + RR->mc->addMultiple(tPtr,RR->node->now(),networkId,mg,field(offset,count * 5),count,totalKnown); } } } break; @@ -535,9 +540,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP default: break; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false,networkId); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_OK,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_OK,"unexpected exception"); } return true; } @@ -573,9 +578,9 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_WHOIS,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_WHOIS,"unexpected exception"); } return true; } @@ -599,9 +604,9 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const } } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_RENDEZVOUS,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_RENDEZVOUS,"unexpected exception"); } return true; } @@ -625,14 +630,14 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar } } else { _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - RR->t->networkAccessDenied(network,_path,packetId(),size(),peer->address(),Packet::VERB_FRAME,true); + RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_FRAME,true); } } else { _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished,nwid); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_FRAME,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_FRAME,"unexpected exception"); } return true; } @@ -654,9 +659,9 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const } if (!network->gate(tPtr,peer)) { - RR->t->networkAccessDenied(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,true); + RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,true); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); return true; } @@ -668,7 +673,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); if ((!from)||(from == network->mac())) { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } @@ -678,20 +683,20 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (to.isMulticast()) { if (network->config().multicastLimit == 0) { - RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } else if (!network->config().permitsBridging(RR->identity.address())) { - RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } @@ -711,12 +716,12 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); } else { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); } } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_EXT_FRAME,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_EXT_FRAME,"unexpected exception"); } return true; } @@ -736,9 +741,9 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false,0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_ECHO,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_ECHO,"unexpected exception"); } return true; } @@ -784,9 +789,9 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,c } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_LIKE,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_LIKE,"unexpected exception"); } return true; } @@ -803,12 +808,13 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t Revocation revocation; CertificateOfOwnership coo; bool trustEstablished = false; + SharedPtr network; unsigned int p = ZT_PACKET_IDX_PAYLOAD; while ((p < size())&&((*this)[p] != 0)) { p += com.deserialize(*this,p); if (com) { - const SharedPtr network(RR->node->network(com.networkId())); + network = RR->node->network(com.networkId()); if (network) { switch (network->addCredential(tPtr,com)) { case Membership::ADD_REJECTED: @@ -829,7 +835,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t const unsigned int numCapabilities = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(cap.networkId())); + if ((!network)||(network->id() != cap.networkId())) + network = RR->node->network(cap.networkId()); if (network) { switch (network->addCredential(tPtr,cap)) { case Membership::ADD_REJECTED: @@ -849,7 +856,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t const unsigned int numTags = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(tag.networkId())); + if ((!network)||(network->id() != tag.networkId())) + network = RR->node->network(tag.networkId()); if (network) { switch (network->addCredential(tPtr,tag)) { case Membership::ADD_REJECTED: @@ -869,7 +877,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t const unsigned int numRevocations = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(revocation.networkId())); + if ((!network)||(network->id() != revocation.networkId())) + network = RR->node->network(revocation.networkId()); if (network) { switch(network->addCredential(tPtr,peer->address(),revocation)) { case Membership::ADD_REJECTED: @@ -889,7 +898,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t const unsigned int numCoos = at(p); p += 2; for(unsigned int i=0;i network(RR->node->network(coo.networkId())); + if ((!network)||(network->id() != coo.networkId())) + network = RR->node->network(coo.networkId()); if (network) { switch(network->addCredential(tPtr,coo)) { case Membership::ADD_REJECTED: @@ -905,9 +915,9 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_NETWORK_CREDENTIALS,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_NETWORK_CREDENTIALS,"unexpected exception"); } return true; } @@ -934,9 +944,9 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false,nwid); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_NETWORK_CONFIG_REQUEST,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_NETWORK_CONFIG_REQUEST,"unexpected exception"); } return true; } @@ -957,9 +967,9 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,c _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false,(network) ? network->id() : 0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_NETWORK_CONFIG,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_NETWORK_CONFIG,"unexpected exception"); } return true; } @@ -1003,9 +1013,9 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished,nwid); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_GATHER,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_GATHER,"unexpected exception"); } return true; } @@ -1030,9 +1040,9 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, } if (!network->gate(tPtr,peer)) { - RR->t->networkAccessDenied(network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,true); + RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,true); _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); return true; } @@ -1055,20 +1065,20 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); if (network->config().multicastLimit == 0) { - RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); return true; } if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) { if (!to.mac().isMulticast()) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_FRAME,"destination not multicast"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } if ((!from)||(from.isMulticast())||(from == network->mac())) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } @@ -1076,8 +1086,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - RR->t->networkFrameDropped(network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } @@ -1101,13 +1111,13 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); } else { _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); } } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_MULTICAST_FRAME,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"unexpected exception"); } return true; } @@ -1119,7 +1129,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); return true; } @@ -1172,9 +1182,9 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt ptr += addrLen; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_PUSH_DIRECT_PATHS,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_PUSH_DIRECT_PATHS,"unexpected exception"); } return true; } @@ -1190,9 +1200,9 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false,0); } catch ( ... ) { - RR->t->incomingPacketInvalid(_path,packetId(),source(),Packet::VERB_USER_MESSAGE,"unexpected exception"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_USER_MESSAGE,"unexpected exception"); } return true; } diff --git a/node/Membership.cpp b/node/Membership.cpp index be6ea6a5..a1453307 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -129,13 +129,13 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme { const uint64_t newts = com.timestamp(); if (newts <= _comRevocationThreshold) { - RR->t->credentialRejected(com,"revoked"); + RR->t->credentialRejected(tPtr,com,"revoked"); return ADD_REJECTED; } const uint64_t oldts = _com.timestamp(); if (newts < oldts) { - RR->t->credentialRejected(com,"old"); + RR->t->credentialRejected(tPtr,com,"old"); return ADD_REJECTED; } if ((newts == oldts)&&(_com == com)) @@ -143,11 +143,11 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme switch(com.verify(RR,tPtr)) { default: - RR->t->credentialRejected(com,"invalid"); + RR->t->credentialRejected(tPtr,com,"invalid"); return ADD_REJECTED; case 0: _com = com; - RR->t->credentialAccepted(com); + RR->t->credentialAccepted(tPtr,com); return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -161,7 +161,7 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot C *rc = remoteCreds.get(cred.id()); if (rc) { if (rc->timestamp() > cred.timestamp()) { - RR->t->credentialRejected(cred,"old"); + RR->t->credentialRejected(tPtr,cred,"old"); return Membership::ADD_REJECTED; } if (*rc == cred) @@ -170,16 +170,16 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot const uint64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); if ((rt)&&(*rt >= cred.timestamp())) { - RR->t->credentialRejected(cred,"revoked"); + RR->t->credentialRejected(tPtr,cred,"revoked"); return Membership::ADD_REJECTED; } switch(cred.verify(RR,tPtr)) { default: - RR->t->credentialRejected(cred,"invalid"); + RR->t->credentialRejected(tPtr,cred,"invalid"); return Membership::ADD_REJECTED; case 0: - RR->t->credentialAccepted(cred); + RR->t->credentialAccepted(tPtr,cred); if (!rc) rc = &(remoteCreds[cred.id()]); *rc = cred; @@ -198,14 +198,14 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme uint64_t *rt; switch(rev.verify(RR,tPtr)) { default: - RR->t->credentialRejected(rev,"invalid"); + RR->t->credentialRejected(tPtr,rev,"invalid"); return ADD_REJECTED; case 0: { const Credential::Type ct = rev.type(); switch(ct) { case Credential::CREDENTIAL_TYPE_COM: if (rev.threshold() > _comRevocationThreshold) { - RR->t->credentialAccepted(rev); + RR->t->credentialAccepted(tPtr,rev); _comRevocationThreshold = rev.threshold(); return ADD_ACCEPTED_NEW; } @@ -221,7 +221,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } return ADD_ACCEPTED_REDUNDANT; default: - RR->t->credentialRejected(rev,"invalid"); + RR->t->credentialRejected(tPtr,rev,"invalid"); return ADD_REJECTED; } } diff --git a/node/Membership.hpp b/node/Membership.hpp index 5e4475da..c6e2b803 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -110,6 +110,11 @@ public: return nconf.com.agreesWith(_com); } + inline bool recentlyAssociated(const uint64_t now) const + { + return ((_com)&&((now - _com.timestamp()) < ZT_PEER_ACTIVITY_TIMEOUT)); + } + /** * Check whether the peer represented by this Membership owns a given resource * diff --git a/node/Network.cpp b/node/Network.cpp index 575b0170..f7b144e3 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -648,7 +648,7 @@ bool Network::filterOutgoingPacket( case DOZTFILTER_DROP: if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(Trace::RuleResultLog *)0,(Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); + RR->t->networkFilter(tPtr,*this,rrl,(Trace::RuleResultLog *)0,(Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); return false; case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() @@ -695,16 +695,16 @@ bool Network::filterOutgoingPacket( RR->sw->send(tPtr,outp,true); if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); + RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); return false; // DROP locally, since we redirected } else { if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,1); + RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,1); return true; } } else { if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); + RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0); return false; } } @@ -775,7 +775,7 @@ int Network::filterIncomingPacket( case DOZTFILTER_DROP: if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(Trace::RuleResultLog *)0,(Capability *)0,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,0); + RR->t->networkFilter(tPtr,*this,rrl,(Trace::RuleResultLog *)0,(Capability *)0,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,0); return 0; // DROP case DOZTFILTER_REDIRECT: // interpreted as ACCEPT but ztFinalDest will have been changed in _doZtFilter() @@ -816,13 +816,13 @@ int Network::filterIncomingPacket( RR->sw->send(tPtr,outp,true); if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(c) ? &crrl : (Trace::RuleResultLog *)0,c,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,0); + RR->t->networkFilter(tPtr,*this,rrl,(c) ? &crrl : (Trace::RuleResultLog *)0,c,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,0); return 0; // DROP locally, since we redirected } } if (_config.remoteTraceTarget) - RR->t->networkFilter(*this,rrl,(c) ? &crrl : (Trace::RuleResultLog *)0,c,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,accept); + RR->t->networkFilter(tPtr,*this,rrl,(c) ? &crrl : (Trace::RuleResultLog *)0,c,sourcePeer->address(),ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,false,true,accept); return accept; } @@ -1135,7 +1135,7 @@ void Network::requestConfiguration(void *tPtr) rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0); rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION); - RR->t->networkConfigRequestSent(*this,ctrl); + RR->t->networkConfigRequestSent(tPtr,*this,ctrl); if (ctrl == RR->identity.address()) { if (RR->localNetworkController) { @@ -1183,6 +1183,13 @@ bool Network::gate(void *tPtr,const SharedPtr &peer) return false; } +bool Network::recentlyAssociatedWith(const Address &addr) +{ + Mutex::Lock _l(_lock); + const Membership *m = _memberships.get(addr); + return ((m)&&(m->recentlyAssociated(RR->node->now()))); +} + void Network::clean() { const uint64_t now = RR->node->now(); diff --git a/node/Network.hpp b/node/Network.hpp index 454a3f20..be5f1a12 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -248,6 +248,19 @@ public: */ bool gate(void *tPtr,const SharedPtr &peer); + /** + * Check whether a given peer has recently had an association with this network + * + * This checks whether a peer has communicated with us recently about this + * network and has possessed a valid certificate of membership. This may return + * true even if the peer has been offline for a while or no longer has a valid + * certificate of membership but had one recently. + * + * @param addr Peer address + * @return True if peer has recently associated + */ + bool recentlyAssociatedWith(const Address &addr); + /** * Do periodic cleanup and housekeeping tasks */ diff --git a/node/Node.cpp b/node/Node.cpp index c54ca450..073af4bd 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -90,14 +90,15 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 } } - idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; if (n <= 0) { RR->identity.generate(); + idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; RR->identity.toString(false,RR->publicIdentityStr); RR->identity.toString(true,RR->secretIdentityStr); stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr,(unsigned int)strlen(RR->secretIdentityStr)); stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } else { + idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; @@ -201,7 +202,7 @@ public: for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET) { - p->sendHELLO(_tPtr,InetAddress(),addr,_now,0); + p->sendHELLO(_tPtr,-1,addr,_now,0); contacted = true; break; } @@ -211,7 +212,7 @@ public: for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET6) { - p->sendHELLO(_tPtr,InetAddress(),addr,_now,0); + p->sendHELLO(_tPtr,-1,addr,_now,0); contacted = true; break; } diff --git a/node/Node.hpp b/node/Node.hpp index 57b99fe9..e60da1ad 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -257,6 +257,8 @@ public: virtual void ncSendRevocation(const Address &destination,const Revocation &rev); virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode); + inline const Address &remoteTraceTarget() const { return _remoteTraceTarget; } + private: RuntimeEnvironment _RR; RuntimeEnvironment *RR; @@ -278,6 +280,7 @@ private: Mutex _backgroundTasksLock; + Address _remoteTraceTarget; uint64_t _now; uint64_t _lastPingCheck; uint64_t _lastHousekeepingRun; diff --git a/node/Packet.hpp b/node/Packet.hpp index 4941e96a..a1ea73e1 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -967,9 +967,8 @@ public: /** * A trace for remote debugging or diagnostics: - * <[8] 64-bit instance ID> - * <[2] 16-bit length of Dictionary> - * <[...] dictionary containing trace information> + * <[...] null-terminated dictionary containing trace information> + * [<[...] additional null-terminated dictionaries>] * * This message contains a remote trace event. Remote trace events can * be sent to observers configured at the network level for those that diff --git a/node/Peer.cpp b/node/Peer.cpp index 79a4bc90..d362be9f 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -72,7 +72,8 @@ void Peer::received( const Packet::Verb verb, const uint64_t inRePacketId, const Packet::Verb inReVerb, - const bool trustEstablished) + const bool trustEstablished, + const uint64_t networkId) { const uint64_t now = RR->node->now(); @@ -183,11 +184,11 @@ void Peer::received( if (replacablePath) { if (verb == Packet::VERB_OK) { - RR->t->peerLearnedNewPath(*this,replacablePath->p,path,packetId); + RR->t->peerLearnedNewPath(tPtr,networkId,*this,replacablePath->p,path,packetId); replacablePath->lr = now; replacablePath->p = path; } else { - RR->t->peerConfirmingUnknownPath(*this,path,packetId,verb); + RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); path->sent(now); } @@ -438,7 +439,7 @@ void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remo } } - RR->t->peerRedirected(*this,op,np); + RR->t->peerRedirected(tPtr,0,*this,op,np); } } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index b24318ec..6d00e3e6 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -102,6 +102,7 @@ public: * @param inRePacketId Packet ID in reply to (default: none) * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP) * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established + * @param networkId Network ID if this pertains to a network, or 0 otherwise */ void received( void *tPtr, @@ -111,7 +112,8 @@ public: const Packet::Verb verb, const uint64_t inRePacketId, const Packet::Verb inReVerb, - const bool trustEstablished); + const bool trustEstablished, + const uint64_t networkId); /** * @param now Current time diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 173230fb..cdbb6303 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -82,7 +82,7 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receive if ( (trusted) && ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) { // Changes to external surface reported by trusted peers causes path reset in this scope - RR->t->resettingPathsInScope(reporter,reporterPhysicalAddress,myPhysicalAddress,scope); + RR->t->resettingPathsInScope(tPtr,reporter,reporterPhysicalAddress,myPhysicalAddress,scope); entry.mySurface = myPhysicalAddress; entry.ts = now; diff --git a/node/Switch.cpp b/node/Switch.cpp index 2fbd243b..eee49775 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -318,7 +318,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const bool fromBridged; if ((fromBridged = (from != network->mac()))) { if (!network->config().permitsBridging(RR->identity.address())) { - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"not a bridge"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"not a bridge"); return; } } @@ -340,7 +340,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const multicastGroup = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(((const unsigned char *)data) + 24,4,0)); } else if (!network->config().enableBroadcast()) { // Don't transmit broadcasts if this network doesn't want them - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"broadcast disabled"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"broadcast disabled"); return; } } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(len >= (40 + 8 + 16))) { @@ -428,7 +428,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const // Check this after NDP emulation, since that has to be allowed in exactly this case if (network->config().multicastLimit == 0) { - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"multicast disabled"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"multicast disabled"); return; } @@ -441,7 +441,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates. if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -467,7 +467,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const SharedPtr toPeer(RR->topology->getPeer(tPtr,toZT)); if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId)) { - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -499,7 +499,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const // for each ZT destination are also done below. This is the same rationale // and design as for multicast. if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId)) { - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked"); return; } @@ -548,7 +548,7 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const outp.compress(); send(tPtr,outp,true); } else { - RR->t->outgoingFrameDropped(network,from,to,etherType,vlanId,len,"filter blocked (bridge replication)"); + RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked (bridge replication)"); } } } @@ -646,7 +646,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) if (_trySend(tPtr,txi->packet,txi->encrypt)) _txQueue.erase(txi++); else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { - RR->t->txTimedOut(txi->dest); + RR->t->txTimedOut(tPtr,txi->dest); _txQueue.erase(txi++); } else ++txi; } diff --git a/node/Trace.cpp b/node/Trace.cpp index 6b68cfe7..dc5ecf19 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -29,16 +29,25 @@ #include "Switch.hpp" #include "Node.hpp" #include "Utils.hpp" +#include "Dictionary.hpp" +#include "CertificateOfMembership.hpp" +#include "CertificateOfOwnership.hpp" +#include "CertificateOfRepresentation.hpp" +#include "Tag.hpp" +#include "Capability.hpp" +#include "Revocation.hpp" namespace ZeroTier { +// Defining ZT_TRACE causes debug tracing messages to be dumped to stderr #ifdef ZT_TRACE + static const char *packetVerbString(Packet::Verb v) { switch(v) { case Packet::VERB_NOP: return "NOP"; case Packet::VERB_HELLO: return "HELLO"; - case Packet::Packet::VERB_ERROR: return "ERROR"; + case Packet::VERB_ERROR: return "ERROR"; case Packet::VERB_OK: return "OK"; case Packet::VERB_WHOIS: return "WHOIS"; case Packet::VERB_RENDEZVOUS: return "RENDEZVOUS"; @@ -73,61 +82,220 @@ static const char *packetErrorString(Packet::ErrorCode e) } return "(unknown)"; } -#endif -void Trace::resettingPathsInScope(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope) +#define TRprintf(f,...) { fprintf(stderr,(f),__VA_ARGS__); fflush(stderr); } + +#else + +#define TRprintf(f,...) + +#endif // ZT_TRACE + +#define ZT_REMOTE_TRACE_FIELD__EVENT "E" +#define ZT_REMOTE_TRACE_FIELD__PACKET_ID "pid" +#define ZT_REMOTE_TRACE_FIELD__PACKET_VERB "pv" +#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "ptpid" +#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_APPROVED "ptpok" +#define ZT_REMOTE_TRACE_FIELD__PACKET_HOPS "phops" +#define ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR "oldrphy" +#define ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR "rzt" +#define ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR "rphy" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_ZTADDR "lzt" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR "lphy" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET "ls" +#define ZT_REMOTE_TRACE_FIELD__IP_SCOPE "ipsc" +#define ZT_REMOTE_TRACE_FIELD__NETWORK_ID "nwid" +#define ZT_REMOTE_TRACE_FIELD__SOURCE_MAC "seth" +#define ZT_REMOTE_TRACE_FIELD__DEST_MAC "deth" +#define ZT_REMOTE_TRACE_FIELD__ETHERTYPE "et" +#define ZT_REMOTE_TRACE_FIELD__VLAN_ID "vlan" +#define ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH "fl" +#define ZT_REMOTE_TRACE_FIELD__FRAME_DATA "fd" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE "credtype" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID "credid" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP "credts" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO "credinfo" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO "crediss" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET "credRt" +#define ZT_REMOTE_TRACE_FIELD__REASON "reason" + +#define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S "1000" +#define ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT_S "1001" +#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S "1002" +#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S "1003" +#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S "1004" +#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S "1005" +#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S "1006" +#define ZT_REMOTE_TRACE_EVENT__DROPPED_HELLO_S "1006" + +#define ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S "2000" +#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S "2001" +#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S "2002" +#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S "2003" +#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S "2004" + +void Trace::resettingPathsInScope(void *const tPtr,const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,reporter); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,reporterPhysicalAddress.toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR,myPhysicalAddress.toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__IP_SCOPE,(uint64_t)scope); + _send(tPtr,d,0); } -void Trace::txTimedOut(const Address &destination) +void Trace::txTimedOut(void *const tPtr,const Address &destination) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT_S); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,destination); + _send(tPtr,d,0); } -void Trace::peerConfirmingUnknownPath(Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb) +void Trace::peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address()); + if (path) { + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + } + _send(tPtr,d,networkId); } -void Trace::peerLearnedNewPath(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId) +void Trace::peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address()); + if (oldPath) { + d.add(ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR,oldPath->address().toString(tmp)); + } + if (newPath) { + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,newPath->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,newPath->localSocket()); + } + _send(tPtr,d,networkId); } -void Trace::peerRedirected(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath) +void Trace::peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address()); + if (oldPath) { + d.add(ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR,oldPath->address().toString(tmp)); + } + if (newPath) { + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,newPath->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,newPath->localSocket()); + } + _send(tPtr,d,networkId); } -void Trace::outgoingFrameDropped(const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason) +void Trace::outgoingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason) { + if (!network) return; // sanity check + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S); + d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,sourceMac.toInt()); + d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,destMac.toInt()); + d.add(ZT_REMOTE_TRACE_FIELD__ETHERTYPE,(uint64_t)etherType); + d.add(ZT_REMOTE_TRACE_FIELD__VLAN_ID,(uint64_t)vlanId); + d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,network); } -void Trace::incomingPacketTrustedPath(const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) +void Trace::incomingNetworkAccessDenied(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested) { + if (!network) return; // sanity check + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id()); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); + if (path) { + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + } } -void Trace::incomingPacketMessageAuthenticationFailure(const SharedPtr &path,const uint64_t packetId,const Address &source) +void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac) { + //Dictionary d; + //d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S); } -void Trace::incomingPacketInvalid(const SharedPtr &path,const uint64_t packetId,const Address &source,const Packet::Verb verb,const char *reason) +void Trace::incomingPacketTrustedPath(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) { + // TODO } -void Trace::incomingPacketDroppedHELLO(const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason) +void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_HOPS,(uint64_t)hops); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + _send(tPtr,d,0); } -void Trace::networkAccessDenied(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested) +void Trace::incomingPacketInvalid(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const Packet::Verb verb,const char *reason) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_HOPS,(uint64_t)hops); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } -void Trace::networkFrameDropped(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac) +void Trace::incomingPacketDroppedHELLO(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason) { + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } -void Trace::networkConfigRequestSent(const Network &network,const Address &controller) +void Trace::networkConfigRequestSent(void *const tPtr,const Network &network,const Address &controller) { } void Trace::networkFilter( + void *const tPtr, const Network &network, const RuleResultLog &primaryRuleSetLog, const RuleResultLog *const matchingCapabilityRuleSetLog, @@ -144,54 +312,185 @@ void Trace::networkFilter( const bool inbound, const int accept) { + //char tmp[128]; + //Dictionary d; + //_send(tPtr,d,network.id()); +} + +void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason) +{ + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); } -void Trace::credentialRejected(const CertificateOfMembership &c,const char *reason) +void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); } -void Trace::credentialRejected(const CertificateOfOwnership &c,const char *reason) +void Trace::credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); } -void Trace::credentialRejected(const CertificateOfRepresentation &c,const char *reason) +void Trace::credentialRejected(void *const tPtr,const Capability &c,const char *reason) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); } -void Trace::credentialRejected(const Capability &c,const char *reason) +void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); } -void Trace::credentialRejected(const Tag &c,const char *reason) +void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *reason) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); } -void Trace::credentialRejected(const Revocation &c,const char *reason) +void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); } -void Trace::credentialAccepted(const CertificateOfMembership &c) +void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); } -void Trace::credentialAccepted(const CertificateOfOwnership &c) +void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); } -void Trace::credentialAccepted(const CertificateOfRepresentation &c) +void Trace::credentialAccepted(void *const tPtr,const Capability &c) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); } -void Trace::credentialAccepted(const Capability &c) +void Trace::credentialAccepted(void *const tPtr,const Tag &c) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); } -void Trace::credentialAccepted(const Tag &c) +void Trace::credentialAccepted(void *const tPtr,const Revocation &c) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); + d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); } -void Trace::credentialAccepted(const Revocation &c) +void Trace::_send(void *const tPtr,const Dictionary &d) { + const Address rtt(RR->node->remoteTraceTarget()); + if (rtt) { + Packet outp(rtt,RR->identity.address(),Packet::VERB_REMOTE_TRACE); + outp.appendCString(d.data()); + outp.compress(); + RR->sw->send(tPtr,outp,true); + } +} + +void Trace::_send(void *const tPtr,const Dictionary &d,const uint64_t networkId) +{ + _send(tPtr,d); + if (networkId) { + const SharedPtr network(RR->node->network(networkId)); + if ((network)&&(network->config().remoteTraceTarget)) { + Packet outp(network->config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE); + outp.appendCString(d.data()); + outp.compress(); + RR->sw->send(tPtr,outp,true); + } + } +} + +void Trace::_send(void *const tPtr,const Dictionary &d,const SharedPtr &network) +{ + _send(tPtr,d); + if ((network)&&(network->config().remoteTraceTarget)) { + Packet outp(network->config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE); + outp.appendCString(d.data()); + outp.compress(); + RR->sw->send(tPtr,outp,true); + } } } // namespace ZeroTier diff --git a/node/Trace.hpp b/node/Trace.hpp index 65d1acf1..eefd5359 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -39,6 +39,7 @@ #include "Packet.hpp" #include "Credential.hpp" #include "InetAddress.hpp" +#include "Dictionary.hpp" namespace ZeroTier { @@ -100,25 +101,25 @@ public: Trace(const RuntimeEnvironment *renv) : RR(renv) {} - void resettingPathsInScope(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope); - void txTimedOut(const Address &destination); + void resettingPathsInScope(void *const tPtr,const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope); + void txTimedOut(void *const tPtr,const Address &destination); - void peerConfirmingUnknownPath(Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb); - void peerLearnedNewPath(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId); - void peerRedirected(Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath); + void peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb); + void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId); + void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath); - void outgoingFrameDropped(const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason); + void incomingPacketTrustedPath(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved); + void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops); + void incomingPacketInvalid(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const Packet::Verb verb,const char *reason); + void incomingPacketDroppedHELLO(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason); - void incomingPacketTrustedPath(const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved); - void incomingPacketMessageAuthenticationFailure(const SharedPtr &path,const uint64_t packetId,const Address &source); - void incomingPacketInvalid(const SharedPtr &path,const uint64_t packetId,const Address &source,const Packet::Verb verb,const char *reason); - void incomingPacketDroppedHELLO(const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason); + void outgoingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason); + void incomingNetworkAccessDenied(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested); + void incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac); - void networkAccessDenied(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested); - void networkFrameDropped(const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac); - - void networkConfigRequestSent(const Network &network,const Address &controller); + void networkConfigRequestSent(void *const tPtr,const Network &network,const Address &controller); void networkFilter( + void *const tPtr, const Network &network, const RuleResultLog &primaryRuleSetLog, const RuleResultLog *const matchingCapabilityRuleSetLog, @@ -135,21 +136,25 @@ public: const bool inbound, const int accept); - void credentialRejected(const CertificateOfMembership &c,const char *reason); - void credentialRejected(const CertificateOfOwnership &c,const char *reason); - void credentialRejected(const CertificateOfRepresentation &c,const char *reason); - void credentialRejected(const Capability &c,const char *reason); - void credentialRejected(const Tag &c,const char *reason); - void credentialRejected(const Revocation &c,const char *reason); - void credentialAccepted(const CertificateOfMembership &c); - void credentialAccepted(const CertificateOfOwnership &c); - void credentialAccepted(const CertificateOfRepresentation &c); - void credentialAccepted(const Capability &c); - void credentialAccepted(const Tag &c); - void credentialAccepted(const Revocation &c); + void credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason); + void credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason); + void credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason); + void credentialRejected(void *const tPtr,const Capability &c,const char *reason); + void credentialRejected(void *const tPtr,const Tag &c,const char *reason); + void credentialRejected(void *const tPtr,const Revocation &c,const char *reason); + void credentialAccepted(void *const tPtr,const CertificateOfMembership &c); + void credentialAccepted(void *const tPtr,const CertificateOfOwnership &c); + void credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c); + void credentialAccepted(void *const tPtr,const Capability &c); + void credentialAccepted(void *const tPtr,const Tag &c); + void credentialAccepted(void *const tPtr,const Revocation &c); private: const RuntimeEnvironment *const RR; + + void _send(void *const tPtr,const Dictionary &d); + void _send(void *const tPtr,const Dictionary &d,const uint64_t networkId); + void _send(void *const tPtr,const Dictionary &d,const SharedPtr &network); }; } // namespace ZeroTier -- cgit v1.2.3 From ba6fd168235fac1c2ec029a49616510da56efe0f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Jul 2017 15:08:57 -0700 Subject: More tracing stuff. --- include/ZeroTierOne.h | 59 ++++++++++++++++++++++ node/Identity.cpp | 32 ++++++++---- node/Node.cpp | 13 ++--- node/Trace.cpp | 129 +++++++++++++------------------------------------ node/Trace.hpp | 4 ++ node/Utils.hpp | 18 +++---- osdep/Binder.hpp | 89 +++++++++++++++++++++++----------- selftest.cpp | 40 ++++++--------- service/OneService.cpp | 11 +++-- 9 files changed, 216 insertions(+), 179 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index f7681768..e4c39fbc 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -257,6 +257,65 @@ extern "C" { */ #define ZT_RULE_PACKET_CHARACTERISTICS_TCP_FIN 0x0000000000000001ULL +// Fields in remote trace dictionaries +#define ZT_REMOTE_TRACE_FIELD__EVENT "E" +#define ZT_REMOTE_TRACE_FIELD__PACKET_ID "pid" +#define ZT_REMOTE_TRACE_FIELD__PACKET_VERB "pv" +#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "ptpid" +#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_APPROVED "ptpok" +#define ZT_REMOTE_TRACE_FIELD__PACKET_HOPS "phops" +#define ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR "oldrphy" +#define ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR "rzt" +#define ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR "rphy" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_ZTADDR "lzt" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR "lphy" +#define ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET "ls" +#define ZT_REMOTE_TRACE_FIELD__IP_SCOPE "ipsc" +#define ZT_REMOTE_TRACE_FIELD__NETWORK_ID "nwid" +#define ZT_REMOTE_TRACE_FIELD__SOURCE_MAC "seth" +#define ZT_REMOTE_TRACE_FIELD__DEST_MAC "deth" +#define ZT_REMOTE_TRACE_FIELD__ETHERTYPE "et" +#define ZT_REMOTE_TRACE_FIELD__VLAN_ID "vlan" +#define ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH "fl" +#define ZT_REMOTE_TRACE_FIELD__FRAME_DATA "fd" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE "crtype" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID "crid" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP "crts" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO "crinfo" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO "criss" +#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET "crrevt" +#define ZT_REMOTE_TRACE_FIELD__REASON "reason" + +// Event types in remote traces +#define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE 0x1000 +#define ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT 0x1001 +#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH 0x1002 +#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH 0x1003 +#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED 0x1004 +#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE 0x1005 +#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID 0x1006 +#define ZT_REMOTE_TRACE_EVENT__DROPPED_HELLO 0x1006 +#define ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED 0x2000 +#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED 0x2001 +#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED 0x2002 +#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED 0x2003 +#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED 0x2004 + +// Event types in remote traces in hex string form +#define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S "1000" +#define ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT_S "1001" +#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S "1002" +#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S "1003" +#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S "1004" +#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S "1005" +#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S "1006" +#define ZT_REMOTE_TRACE_EVENT__DROPPED_HELLO_S "1006" +#define ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S "2000" +#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S "2001" +#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S "2002" +#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S "2003" +#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S "2004" + /****************************************************************************/ /* Structures and other types */ /****************************************************************************/ diff --git a/node/Identity.cpp b/node/Identity.cpp index dba27d1c..a972d60d 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -157,44 +157,58 @@ char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_ bool Identity::fromString(const char *str) { - if (!str) + if (!str) { + _address.zero(); return false; - - char *saveptr = (char *)0; + } char tmp[ZT_IDENTITY_STRING_BUFFER_LENGTH]; - if (!Utils::scopy(tmp,sizeof(tmp),str)) + if (!Utils::scopy(tmp,sizeof(tmp),str)) { + _address.zero(); return false; + } delete _privateKey; _privateKey = (C25519::Private *)0; int fno = 0; + char *saveptr = (char *)0; for(char *f=Utils::stok(tmp,":",&saveptr);(f);f=Utils::stok((char *)0,":",&saveptr)) { switch(fno++) { case 0: _address = Address(Utils::hexStrToU64(f)); - if (_address.isReserved()) + if (_address.isReserved()) { + _address.zero(); return false; + } break; case 1: - if ((f[0] != '0')||(f[1])) + if ((f[0] != '0')||(f[1])) { + _address.zero(); return false; + } break; case 2: - if (Utils::unhex(f,_publicKey.data,(unsigned int)_publicKey.size()) != _publicKey.size()) + if (Utils::unhex(f,_publicKey.data,(unsigned int)_publicKey.size()) != _publicKey.size()) { + _address.zero(); return false; + } break; case 3: _privateKey = new C25519::Private(); - if (Utils::unhex(f,_privateKey->data,(unsigned int)_privateKey->size()) != _privateKey->size()) + if (Utils::unhex(f,_privateKey->data,(unsigned int)_privateKey->size()) != _privateKey->size()) { + _address.zero(); return false; + } break; default: + _address.zero(); return false; } } - if (fno < 3) + if (fno < 3) { + _address.zero(); return false; + } return true; } diff --git a/node/Node.cpp b/node/Node.cpp index 073af4bd..ff3acfc2 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -78,7 +78,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 uint64_t idtmp[2]; idtmp[0] = 0; idtmp[1] = 0; - char tmp[1024]; + char tmp[2048]; int n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,tmp,sizeof(tmp) - 1); if (n > 0) { tmp[n] = (char)0; @@ -92,21 +92,18 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 if (n <= 0) { RR->identity.generate(); - idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; RR->identity.toString(false,RR->publicIdentityStr); RR->identity.toString(true,RR->secretIdentityStr); + idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_SECRET,idtmp,RR->secretIdentityStr,(unsigned int)strlen(RR->secretIdentityStr)); stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } else { idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); - if (n > 0) { - tmp[n] = (char)0; - if (RR->publicIdentityStr != tmp) - n = -1; + if ((n > 0)&&(n < sizeof(RR->publicIdentityStr))&&(n < sizeof(tmp))) { + if (memcmp(tmp,RR->publicIdentityStr,n)) + stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } - if (n <= 0) - stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } try { diff --git a/node/Trace.cpp b/node/Trace.cpp index dc5ecf19..21d06228 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -39,101 +39,6 @@ namespace ZeroTier { -// Defining ZT_TRACE causes debug tracing messages to be dumped to stderr -#ifdef ZT_TRACE - -static const char *packetVerbString(Packet::Verb v) -{ - switch(v) { - case Packet::VERB_NOP: return "NOP"; - case Packet::VERB_HELLO: return "HELLO"; - case Packet::VERB_ERROR: return "ERROR"; - case Packet::VERB_OK: return "OK"; - case Packet::VERB_WHOIS: return "WHOIS"; - case Packet::VERB_RENDEZVOUS: return "RENDEZVOUS"; - case Packet::VERB_FRAME: return "FRAME"; - case Packet::VERB_EXT_FRAME: return "EXT_FRAME"; - case Packet::VERB_ECHO: return "ECHO"; - case Packet::VERB_MULTICAST_LIKE: return "MULTICAST_LIKE"; - case Packet::VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS"; - case Packet::VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST"; - case Packet::VERB_NETWORK_CONFIG: return "NETWORK_CONFIG"; - case Packet::VERB_MULTICAST_GATHER: return "MULTICAST_GATHER"; - case Packet::VERB_MULTICAST_FRAME: return "MULTICAST_FRAME"; - case Packet::VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS"; - case Packet::VERB_USER_MESSAGE: return "USER_MESSAGE"; - case Packet::VERB_REMOTE_TRACE: return "REMOTE_TRACE"; - } - return "(unknown)"; -} - -static const char *packetErrorString(Packet::ErrorCode e) -{ - switch(e) { - case Packet::ERROR_NONE: return "NONE"; - case Packet::ERROR_INVALID_REQUEST: return "INVALID_REQUEST"; - case Packet::ERROR_BAD_PROTOCOL_VERSION: return "BAD_PROTOCOL_VERSION"; - case Packet::ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND"; - case Packet::ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION"; - case Packet::ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION"; - case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE"; - case Packet::ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED"; - case Packet::ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST"; - } - return "(unknown)"; -} - -#define TRprintf(f,...) { fprintf(stderr,(f),__VA_ARGS__); fflush(stderr); } - -#else - -#define TRprintf(f,...) - -#endif // ZT_TRACE - -#define ZT_REMOTE_TRACE_FIELD__EVENT "E" -#define ZT_REMOTE_TRACE_FIELD__PACKET_ID "pid" -#define ZT_REMOTE_TRACE_FIELD__PACKET_VERB "pv" -#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "ptpid" -#define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_APPROVED "ptpok" -#define ZT_REMOTE_TRACE_FIELD__PACKET_HOPS "phops" -#define ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR "oldrphy" -#define ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR "rzt" -#define ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR "rphy" -#define ZT_REMOTE_TRACE_FIELD__LOCAL_ZTADDR "lzt" -#define ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR "lphy" -#define ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET "ls" -#define ZT_REMOTE_TRACE_FIELD__IP_SCOPE "ipsc" -#define ZT_REMOTE_TRACE_FIELD__NETWORK_ID "nwid" -#define ZT_REMOTE_TRACE_FIELD__SOURCE_MAC "seth" -#define ZT_REMOTE_TRACE_FIELD__DEST_MAC "deth" -#define ZT_REMOTE_TRACE_FIELD__ETHERTYPE "et" -#define ZT_REMOTE_TRACE_FIELD__VLAN_ID "vlan" -#define ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH "fl" -#define ZT_REMOTE_TRACE_FIELD__FRAME_DATA "fd" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE "credtype" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID "credid" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP "credts" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO "credinfo" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO "crediss" -#define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET "credRt" -#define ZT_REMOTE_TRACE_FIELD__REASON "reason" - -#define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S "1000" -#define ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT_S "1001" -#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S "1002" -#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S "1003" -#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S "1004" -#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S "1005" -#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S "1006" -#define ZT_REMOTE_TRACE_EVENT__DROPPED_HELLO_S "1006" - -#define ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S "2000" -#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S "2001" -#define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S "2002" -#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S "2003" -#define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S "2004" - void Trace::resettingPathsInScope(void *const tPtr,const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope) { char tmp[128]; @@ -328,6 +233,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason) @@ -341,6 +247,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c, d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } void Trace::credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason) @@ -352,6 +259,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfRepresentatio d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } void Trace::credentialRejected(void *const tPtr,const Capability &c,const char *reason) @@ -365,6 +273,7 @@ void Trace::credentialRejected(void *const tPtr,const Capability &c,const char * d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) @@ -379,6 +288,7 @@ void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *reason) @@ -391,6 +301,7 @@ void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char * d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,0); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c) @@ -402,6 +313,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + _send(tPtr,d,0); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) @@ -413,6 +325,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + _send(tPtr,d,0); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c) @@ -422,6 +335,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentatio d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); + _send(tPtr,d,0); } void Trace::credentialAccepted(void *const tPtr,const Capability &c) @@ -433,6 +347,7 @@ void Trace::credentialAccepted(void *const tPtr,const Capability &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); + _send(tPtr,d,0); } void Trace::credentialAccepted(void *const tPtr,const Tag &c) @@ -445,6 +360,7 @@ void Trace::credentialAccepted(void *const tPtr,const Tag &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); + _send(tPtr,d,0); } void Trace::credentialAccepted(void *const tPtr,const Revocation &c) @@ -455,10 +371,33 @@ void Trace::credentialAccepted(void *const tPtr,const Revocation &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); + _send(tPtr,d,0); } void Trace::_send(void *const tPtr,const Dictionary &d) { +#ifdef ZT_TRACE + unsigned int i = 0; + while (i < (unsigned int)(sizeof(_traceMsgBuf) - 1)) { + const char c = d.data()[i]; + if (c == 0) { + break; + } else if (c == '\n') { + _traceMsgBuf[i++] = ' '; + } else if ((c >= 32)&&(c <= 126)) { + _traceMsgBuf[i++] = c; + } else { + if ((i + 3) < (unsigned int)(sizeof(_traceMsgBuf) - 1)) { + _traceMsgBuf[i++] = '\\'; + Utils::hex((uint8_t)c,_traceMsgBuf + i); + } + } + } + _traceMsgBuf[i] = (char)0; + //printf("%s\n",_traceMsgBuf); + RR->node->postEvent(tPtr,ZT_EVENT_TRACE,_traceMsgBuf); +#endif + const Address rtt(RR->node->remoteTraceTarget()); if (rtt) { Packet outp(rtt,RR->identity.address(),Packet::VERB_REMOTE_TRACE); diff --git a/node/Trace.hpp b/node/Trace.hpp index eefd5359..dae67e28 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -155,6 +155,10 @@ private: void _send(void *const tPtr,const Dictionary &d); void _send(void *const tPtr,const Dictionary &d,const uint64_t networkId); void _send(void *const tPtr,const Dictionary &d,const SharedPtr &network); + +#ifdef ZT_TRACE + char _traceMsgBuf[4096]; +#endif }; } // namespace ZeroTier diff --git a/node/Utils.hpp b/node/Utils.hpp index 1139c9f1..8f61d396 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -150,8 +150,8 @@ public: { char *save = s; for(unsigned int i=0;i(d)[i]; - *(s++) = HEXCHARS[(b >> 4) & 0xf]; + const unsigned int b = reinterpret_cast(d)[i]; + *(s++) = HEXCHARS[b >> 4]; *(s++) = HEXCHARS[b & 0xf]; } *s = (char)0; @@ -162,18 +162,18 @@ public: { unsigned int l = 0; while (l < buflen) { - uint8_t hc = (uint8_t)*(h++); + uint8_t hc = *(reinterpret_cast(h++)); if (!hc) break; uint8_t c = 0; - if ((hc >= 48)&&(hc <= 57)) + if ((hc >= 48)&&(hc <= 57)) // 0..9 c = hc - 48; - else if ((hc >= 97)&&(hc <= 102)) + else if ((hc >= 97)&&(hc <= 102)) // a..f c = hc - 87; - else if ((hc >= 65)&&(hc <= 70)) + else if ((hc >= 65)&&(hc <= 70)) // A..F c = hc - 55; - hc = (uint8_t)*(h++); + hc = *(reinterpret_cast(h++)); if (!hc) break; c <<= 4; @@ -195,7 +195,7 @@ public: const char *hend = h + hlen; while (l < buflen) { if (h == hend) break; - uint8_t hc = (uint8_t)*(h++); + uint8_t hc = *(reinterpret_cast(h++)); if (!hc) break; uint8_t c = 0; @@ -207,7 +207,7 @@ public: c = hc - 55; if (h == hend) break; - hc = (uint8_t)*(h++); + hc = *(reinterpret_cast(h++)); if (!hc) break; c <<= 4; diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index 126dba28..17a0fbf6 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -58,6 +58,7 @@ #include #include #include +#include #include "../node/NonCopyable.hpp" #include "../node/InetAddress.hpp" @@ -70,6 +71,9 @@ // Period between refreshes of bindings #define ZT_BINDER_REFRESH_PERIOD 30000 +// Max number of bindings +#define ZT_BINDER_MAX_BINDINGS 128 + namespace ZeroTier { /** @@ -95,7 +99,7 @@ private: }; public: - Binder() {} + Binder() : _bindingCount(0) {} /** * Close all bound ports, should be called on shutdown @@ -106,10 +110,11 @@ public: void closeAll(Phy &phy) { Mutex::Lock _l(_lock); - for(std::vector<_Binding>::iterator b(_bindings.begin());b!=_bindings.end();++b) { - phy.close(b->udpSock,false); - phy.close(b->tcpListenSock,false); + for(unsigned int b=0,c=_bindingCount;b newBindings; + const unsigned int oldBindingCount = _bindingCount; + _bindingCount = 0; // Save bindings that are still valid, close those that are not - for(std::vector<_Binding>::iterator b(_bindings.begin());b!=_bindings.end();++b) { - if (localIfAddrs.find(b->address) != localIfAddrs.end()) { - newBindings.push_back(*b); + for(unsigned int b=0;budpSock,false); - phy.close(b->tcpListenSock,false); + PhySocket *const udps = _bindings[b].udpSock; + PhySocket *const tcps = _bindings[b].tcpListenSock; + _bindings[b].udpSock = (PhySocket *)0; + _bindings[b].tcpListenSock = (PhySocket *)0; + phy.close(udps,false); + phy.close(tcps,false); } } // Create new bindings for those not already bound for(std::map::const_iterator ii(localIfAddrs.begin());ii!=localIfAddrs.end();++ii) { - typename std::vector<_Binding>::const_iterator bi(newBindings.begin()); - while (bi != newBindings.end()) { - if (bi->address == ii->first) + unsigned int bi = 0; + while (bi != _bindingCount) { + if (_bindings[bi].address == ii->first) break; ++bi; } - if (bi == newBindings.end()) { + if (bi == _bindingCount) { udps = phy.udpBind(reinterpret_cast(&(ii->first)),(void *)0,ZT_UDP_DESIRED_BUF_SIZE); tcps = phy.tcpListen(reinterpret_cast(&(ii->first)),(void *)0); if ((udps)&&(tcps)) { @@ -358,15 +370,18 @@ public: setsockopt(fd,SOL_SOCKET,SO_BINDTODEVICE,tmp,strlen(tmp)); } #endif // __LINUX__ - newBindings.push_back(_Binding()); - newBindings.back().udpSock = udps; - newBindings.back().tcpListenSock = tcps; - newBindings.back().address = ii->first; + if (_bindingCount < ZT_BINDER_MAX_BINDINGS) { + _bindings[_bindingCount].udpSock = udps; + _bindings[_bindingCount].tcpListenSock = tcps; + _bindings[_bindingCount].address = ii->first; + ++_bindingCount; + } + } else { + phy.close(udps,false); + phy.close(tcps,false); } } } - - _bindings.swap(newBindings); } /** @@ -376,8 +391,8 @@ public: { std::vector aa; Mutex::Lock _l(_lock); - for(std::vector<_Binding>::const_iterator b(_bindings.begin());b!=_bindings.end();++b) - aa.push_back(b->address); + for(unsigned int b=0,c=_bindingCount;b::const_iterator b(_bindings.begin());b!=_bindings.end();++b) { - if (ttl) phy.setIp4UdpTtl(b->udpSock,ttl); - if (phy.udpSend(b->udpSock,(const struct sockaddr *)addr,data,len)) r = true; - if (ttl) phy.setIp4UdpTtl(b->udpSock,255); + for(unsigned int b=0,c=_bindingCount;b::const_iterator b(_bindings.begin());b!=_bindings.end();++b) { - if (b->address == addr) + for(unsigned int b=0;b<_bindingCount;++b) { + if (_bindings[b].address == addr) return true; } return false; } + /** + * Quickly check that a UDP socket is valid + * + * @param udpSock UDP socket to check + * @return True if socket is currently bound/allocated + */ + inline bool isUdpSocketValid(PhySocket *const udpSock) + { + for(unsigned int b=0,c=_bindingCount;b _bindings; + _Binding _bindings[ZT_BINDER_MAX_BINDINGS]; + std::atomic _bindingCount; Mutex _lock; }; diff --git a/selftest.cpp b/selftest.cpp index 882422bc..f4de36a9 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -633,33 +633,25 @@ static int testPacket() return 0; } -static void _testExcept(int &depth) -{ - if (depth >= 16) { - throw std::runtime_error("LOL!"); - } else { - ++depth; - _testExcept(depth); - } -} - static int testOther() { char buf[1024]; - - std::cout << "[other] Testing C++ exceptions... "; std::cout.flush(); - int depth = 0; - try { - _testExcept(depth); - } catch (std::runtime_error &e) { - if (depth == 16) { - std::cout << "OK" << std::endl; - } else { - std::cout << "ERROR (depth not 16)" << std::endl; - return -1; - } - } catch ( ... ) { - std::cout << "ERROR (exception not std::runtime_error)" << std::endl; + char buf2[4096]; + char buf3[1024]; + + std::cout << "[other] Testing hex/unhex... "; std::cout.flush(); + Utils::getSecureRandom(buf,(unsigned int)sizeof(buf)); + Utils::hex(buf,(unsigned int)sizeof(buf),buf2); + Utils::unhex(buf2,buf3,(unsigned int)sizeof(buf3)); + if (memcmp(buf,buf3,sizeof(buf)) == 0) { + std::cout << "PASS" << std::endl; + } else { + std::cout << "FAIL!" << std::endl; + buf2[78] = 0; + std::cout << buf2 << std::endl; + Utils::hex(buf3,(unsigned int)sizeof(buf3),buf2); + buf2[78] = 0; + std::cout << buf2 << std::endl; return -1; } diff --git a/service/OneService.cpp b/service/OneService.cpp index 352985c8..1b07eb79 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -658,16 +658,18 @@ public: return _termReason; } - // Bind local control socket + // Bind TCP control socket to 127.0.0.1 and ::1 as well for loopback TCP control socket queries { struct sockaddr_in lo4; memset(&lo4,0,sizeof(lo4)); lo4.sin_family = AF_INET; + lo4.sin_addr.s_addr = Utils::hton((uint32_t)0x7f000001); lo4.sin_port = Utils::hton((uint16_t)_ports[0]); _localControlSocket4 = _phy.tcpListen((const struct sockaddr *)&lo4); struct sockaddr_in6 lo6; memset(&lo6,0,sizeof(lo6)); lo6.sin6_family = AF_INET6; + lo6.sin6_addr.s6_addr[15] = 1; lo6.sin6_port = lo4.sin_port; _localControlSocket6 = _phy.tcpListen((const struct sockaddr *)&lo6); } @@ -1661,12 +1663,11 @@ public: { if ((len >= 16)&&(reinterpret_cast(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) _lastDirectReceiveFromGlobal = OSUtils::now(); - const ZT_ResultCode rc = _node->processWirePacket( (void *)0, OSUtils::now(), - (int64_t)((uintptr_t)sock), - (const struct sockaddr_storage *)from, // Phy<> uses sockaddr_storage, so it'll always be that big + reinterpret_cast(sock), + reinterpret_cast(from), // Phy<> uses sockaddr_storage, so it'll always be that big data, len, &_nextBackgroundTaskDeadline); @@ -2200,7 +2201,7 @@ public: // proxy fallback, which is slow. #endif // ZT_TCP_FALLBACK_RELAY - if ((localSocket != 0)&&(localSocket != -1)) { + if ((localSocket != -1)&&(localSocket != 0)&&(_binder.isUdpSocketValid((PhySocket *)((uintptr_t)localSocket)))) { if ((ttl)&&(addr->ss_family == AF_INET)) _phy.setIp4UdpTtl((PhySocket *)((uintptr_t)localSocket),ttl); const bool r = _phy.udpSend((PhySocket *)((uintptr_t)localSocket),(const struct sockaddr *)addr,data,len); if ((ttl)&&(addr->ss_family == AF_INET)) _phy.setIp4UdpTtl((PhySocket *)((uintptr_t)localSocket),255); -- cgit v1.2.3 From c692f2e740a3d3f1165cbd8ba16b65ed9b923056 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 13 Jul 2017 16:31:16 -0700 Subject: Fix for new identity generation bug. --- node/C25519.cpp | 18 ++---------------- node/Node.cpp | 1 + node/Utils.hpp | 2 +- 3 files changed, 4 insertions(+), 17 deletions(-) (limited to 'node') diff --git a/node/C25519.cpp b/node/C25519.cpp index f35a88c2..4158f1ba 100644 --- a/node/C25519.cpp +++ b/node/C25519.cpp @@ -286,24 +286,10 @@ static inline int crypto_scalarmult(unsigned char *q,const unsigned char *n,cons return 0; } -//static const unsigned char base[32] = {9}; +static const unsigned char base[32] = {9}; static inline int crypto_scalarmult_base(unsigned char *q,const unsigned char *n) { - //return crypto_scalarmult(q,n,base); - unsigned int work[96]; - unsigned char e[32]; - unsigned int i; - for (i = 0;i < 32;++i) e[i] = n[i]; - e[0] &= 248; - e[31] &= 127; - e[31] |= 64; - for (i = 0;i < 32;++i) work[i] = 9; - mainloop(work,e); - recip(work + 32,work + 32); - mult(work + 64,work,work + 32); - freeze(work + 64); - for (i = 0;i < 32;++i) q[i] = work[64 + i]; - return 0; + return crypto_scalarmult(q,n,base); } ////////////////////////////////////////////////////////////////////////////// diff --git a/node/Node.cpp b/node/Node.cpp index ff3acfc2..dea0f5ca 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -117,6 +117,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 delete RR->topology; delete RR->mc; delete RR->sw; + delete RR->t; throw; } diff --git a/node/Utils.hpp b/node/Utils.hpp index 8f61d396..87584fcf 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -148,7 +148,7 @@ public: static inline char *hex(const void *d,unsigned int l,char *s) { - char *save = s; + char *const save = s; for(unsigned int i=0;i(d)[i]; *(s++) = HEXCHARS[b >> 4]; -- cgit v1.2.3 From 4ecc0c59cafac54ff2d32e97b130f83b7481da2e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 14 Jul 2017 13:03:16 -0700 Subject: Plumbing through of remote trace into controller code. --- controller/EmbeddedNetworkController.cpp | 67 ++++++++++++++++++++++++++++++++ controller/EmbeddedNetworkController.hpp | 4 ++ include/ZeroTierOne.h | 45 ++++++++++++++++++++- node/IncomingPacket.cpp | 27 ++++++++++++- node/IncomingPacket.hpp | 1 + node/Packet.hpp | 3 -- service/OneService.cpp | 6 +++ 7 files changed, 148 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index b57a37e8..8b8a93bd 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -621,6 +621,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"],false); if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"],false); + if (b.count("remoteTraceTarget")) { + const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],"")); + if (rtt.length() == 10) { + member["remoteTraceTarget"] = rtt; + } else { + member["remoteTraceTarget"] = json(); + } + } + if (b.count("authorized")) { const bool newAuth = OSUtils::jsonBool(b["authorized"],false); if (newAuth != OSUtils::jsonBool(member["authorized"],false)) { @@ -764,6 +773,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("multicastLimit")) network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"],32ULL); if (b.count("mtu")) network["mtu"] = std::max(std::min((unsigned int)OSUtils::jsonInt(b["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU); + if (b.count("remoteTraceTarget")) { + const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],"")); + if (rtt.length() == 10) { + network["remoteTraceTarget"] = rtt; + } else { + network["remoteTraceTarget"] = json(); + } + } + if (b.count("v4AssignMode")) { json nv4m; json &v4m = b["v4AssignMode"]; @@ -1065,6 +1083,55 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( return 404; } +void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) +{ + // Convert Dictionary into JSON object + json d; + char *saveptr = (char *)0; + for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) { + char *eq = strchr(l,'='); + if (eq > l) { + std::string k(l,(unsigned long)(eq - l)); + std::string v; + ++eq; + while (*eq) { + if (*eq == '\\') { + ++eq; + if (*eq) { + switch(*eq) { + case 'r': + v.push_back('\r'); + break; + case 'n': + v.push_back('\n'); + break; + case '0': + v.push_back((char)0); + break; + case 'e': + v.push_back('='); + break; + default: + v.push_back(*eq); + break; + } + ++eq; + } + } else { + v.push_back(*(eq++)); + } + } + if (v.length() > 0) + d[k] = v; + } + } + + char p[128]; + OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx_%.16llx.json",rt.origin,OSUtils::now()); + _db.writeRaw(p,OSUtils::jsonDump(d)); + //fprintf(stdout,"%s\n",OSUtils::jsonDump(d).c_str()); fflush(stdout); +} + void EmbeddedNetworkController::threadMain() throw() { diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 1589ea71..03ba0b95 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -90,6 +90,8 @@ public: std::string &responseBody, std::string &responseContentType); + void handleRemoteTrace(const ZT_RemoteTrace &rt); + void threadMain() throw(); @@ -142,6 +144,7 @@ private: if (!member.count("vRev")) member["vRev"] = -1; if (!member.count("vProto")) member["vProto"] = -1; if (!member.count("physicalAddr")) member["physicalAddr"] = nlohmann::json(); + if (!member.count("remoteTraceTarget")) member["remoteTraceTarget"] = nlohmann::json(); member["objtype"] = "member"; } inline void _initNetwork(nlohmann::json &network) @@ -159,6 +162,7 @@ private: if (!network.count("routes")) network["routes"] = nlohmann::json::array(); if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU; + if (!network.count("remoteTraceTarget")) network["remoteTraceTarget"] = nlohmann::json(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment network["rules"] = {{ diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e4c39fbc..14ddc7fe 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -470,9 +470,52 @@ enum ZT_Event * * Meta-data: ZT_UserMessage structure */ - ZT_EVENT_USER_MESSAGE = 6 + ZT_EVENT_USER_MESSAGE = 6, + + /** + * Remote trace received + * + * These are generated when a VERB_REMOTE_TRACE is received. Note + * that any node can fling one of these at us. It is your responsibility + * to filter and determine if it's worth paying attention to. If it's + * not just drop it. Most nodes that are not active controllers ignore + * these, and controllers only save them if they pertain to networks + * with remote tracing enabled. + * + * Meta-data: ZT_RemoteTrace structure + */ + ZT_EVENT_REMOTE_TRACE = 7 }; +/** + * Payload of REMOTE_TRACE event + */ +typedef struct +{ + /** + * ZeroTier address of sender + */ + uint64_t origin; + + /** + * Null-terminated Dictionary containing key/value pairs sent by origin + * + * This *should* be a dictionary, but the implementation only checks + * that it is a valid non-empty C-style null-terminated string. Be very + * careful to use a well-tested parser to parse this as it represents + * data received from a potentially un-trusted peer on the network. + * Invalid payloads should be dropped. + * + * The contents of data[] may be modified. + */ + char *data; + + /** + * Length of dict[] in bytes, including terminating null + */ + unsigned int len; +} ZT_RemoteTrace; + /** * User message used with ZT_EVENT_USER_MESSAGE * diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index a5875d1e..5e5d1d72 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1192,7 +1192,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { try { - if (size() >= (ZT_PACKET_IDX_PAYLOAD + 8)) { + if (likely(size() >= (ZT_PACKET_IDX_PAYLOAD + 8))) { ZT_UserMessage um; um.origin = peer->address().toInt(); um.typeId = at(ZT_PACKET_IDX_PAYLOAD); @@ -1207,6 +1207,31 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con return true; } +bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) +{ + ZT_RemoteTrace rt; + try { + const char *ptr = reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD; + const char *const eof = reinterpret_cast(data()) + size(); + rt.origin = peer->address().toInt(); + rt.data = const_cast(ptr); // start of first string + while (ptr < eof) { + if (!*ptr) { // end of string + rt.len = (unsigned int)(ptr - rt.data); + if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) + RR->node->postEvent(tPtr,ZT_EVENT_REMOTE_TRACE,&rt); + rt.data = const_cast(++ptr); // start of next string, if any + } else { + ++ptr; + } + } + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0); + } catch ( ... ) { + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_REMOTE_TRACE,"unexpected exception"); + } + return true; +} + void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid) { const uint64_t now = RR->node->now(); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 11b60712..692c63df 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -139,6 +139,7 @@ private: bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); bool _doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); + bool _doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer); void _sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid); diff --git a/node/Packet.hpp b/node/Packet.hpp index a1ea73e1..b8e69fa9 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -978,9 +978,6 @@ public: * The instance ID is a random 64-bit value generated by each ZeroTier * node on startup. This is helpful in identifying traces from different * members of a cluster. - * - * The Dictionary serialization format is the same as used for network - * configurations. The maximum size of a trace is 10000 bytes. */ VERB_REMOTE_TRACE = 0x15 }; diff --git a/service/OneService.cpp b/service/OneService.cpp index 1b07eb79..115830e5 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -2058,6 +2058,12 @@ public: } } break; + case ZT_EVENT_REMOTE_TRACE: { + const ZT_RemoteTrace *rt = reinterpret_cast(metaData); + if ((rt)&&(rt->len > 0)&&(rt->len <= ZT_MAX_REMOTE_TRACE_SIZE)&&(rt->data)) + _controller->handleRemoteTrace(*rt); + } + default: break; } -- cgit v1.2.3 From d939d8d21d79c23a5af82bcc60214b4e19dc5f74 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 14 Jul 2017 14:57:40 -0700 Subject: A bit more remote tracing stuff. --- include/ZeroTierOne.h | 13 ++++++++++++ node/IncomingPacket.cpp | 10 ++++----- node/Trace.cpp | 56 ++++++++++++++++++++++++++++++++++++++++++------- node/Trace.hpp | 4 ++-- 4 files changed, 68 insertions(+), 15 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 14ddc7fe..b123e8e3 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -272,12 +272,20 @@ extern "C" { #define ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET "ls" #define ZT_REMOTE_TRACE_FIELD__IP_SCOPE "ipsc" #define ZT_REMOTE_TRACE_FIELD__NETWORK_ID "nwid" +#define ZT_REMOTE_TRACE_FIELD__SOURCE_ZTADDR "szt" +#define ZT_REMOTE_TRACE_FIELD__DEST_ZTADDR "dzt" #define ZT_REMOTE_TRACE_FIELD__SOURCE_MAC "seth" #define ZT_REMOTE_TRACE_FIELD__DEST_MAC "deth" #define ZT_REMOTE_TRACE_FIELD__ETHERTYPE "et" #define ZT_REMOTE_TRACE_FIELD__VLAN_ID "vlan" #define ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH "fl" #define ZT_REMOTE_TRACE_FIELD__FRAME_DATA "fd" +#define ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_NOTEE "ffnotee" +#define ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_INBOUND "ffdir" +#define ZT_REMOTE_TRACE_FIELD__FILTER_RESULT "fresult" +#define ZT_REMOTE_TRACE_FIELD__FILTER_BASE_RULE_LOG "frlog" +#define ZT_REMOTE_TRACE_FIELD__FILTER_CAP_RULE_LOG "fclog" +#define ZT_REMOTE_TRACE_FIELD__FILTER_CAP_ID "fcid" #define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE "crtype" #define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID "crid" #define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP "crts" @@ -285,6 +293,7 @@ extern "C" { #define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO "criss" #define ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET "crrevt" #define ZT_REMOTE_TRACE_FIELD__REASON "reason" +#define ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID "nwctrl" // Event types in remote traces #define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE 0x1000 @@ -300,6 +309,8 @@ extern "C" { #define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED 0x2002 #define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED 0x2003 #define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED 0x2004 +#define ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT 0x2005 +#define ZT_REMOTE_TRACE_EVENT__NETWORK_FILTER_TRACE 0x2006 // Event types in remote traces in hex string form #define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S "1000" @@ -315,6 +326,8 @@ extern "C" { #define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S "2002" #define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S "2003" #define ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S "2004" +#define ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT_S "2005" +#define ZT_REMOTE_TRACE_EVENT__NETWORK_FILTER_TRACE_S "2006" /****************************************************************************/ /* Structures and other types */ diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 5e5d1d72..94c73f81 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -683,19 +683,19 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (remote)"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } else if (to != network->mac()) { if (to.isMulticast()) { if (network->config().multicastLimit == 0) { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"multicast disabled"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } } else if (!network->config().permitsBridging(RR->identity.address())) { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to); + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (local)"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } @@ -1065,7 +1065,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); if (network->config().multicastLimit == 0) { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"multicast disabled"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); return true; } @@ -1086,7 +1086,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, if (network->config().permitsBridging(peer->address())) { network->learnBridgeRoute(from,peer->address()); } else { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac()); + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"bridging not allowed (remote)"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } diff --git a/node/Trace.cpp b/node/Trace.cpp index 21d06228..2b1c69dd 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -120,8 +120,9 @@ void Trace::outgoingNetworkFrameDropped(void *const tPtr,const SharedPtrid()); d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); if (path) { d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); } + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id()); + _send(tPtr,d,network); } -void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac) +void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac,const char *reason) { - //Dictionary d; - //d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S); + if (!network) return; // sanity check + char tmp[128]; + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId); + d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb); + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); + if (path) { + d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); + d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + } + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id()); + d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,sourceMac.toInt()); + d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,destMac.toInt()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); + _send(tPtr,d,network); } void Trace::incomingPacketTrustedPath(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) @@ -197,6 +214,11 @@ void Trace::incomingPacketDroppedHELLO(void *const tPtr,const SharedPtr &p void Trace::networkConfigRequestSent(void *const tPtr,const Network &network,const Address &controller) { + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id()); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID,controller); + _send(tPtr,d,0); } void Trace::networkFilter( @@ -217,9 +239,27 @@ void Trace::networkFilter( const bool inbound, const int accept) { - //char tmp[128]; - //Dictionary d; - //_send(tPtr,d,network.id()); + Dictionary d; + d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_FILTER_TRACE_S); + d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id()); + d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_ZTADDR,ztSource); + d.add(ZT_REMOTE_TRACE_FIELD__DEST_ZTADDR,ztDest); + d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,macSource.toInt()); + d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,macDest.toInt()); + d.add(ZT_REMOTE_TRACE_FIELD__ETHERTYPE,(uint64_t)etherType); + d.add(ZT_REMOTE_TRACE_FIELD__VLAN_ID,(uint64_t)vlanId); + d.add(ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_NOTEE,noTee ? "1" : "0"); + d.add(ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_INBOUND,inbound ? "1" : "0"); + d.add(ZT_REMOTE_TRACE_FIELD__FILTER_RESULT,(int64_t)accept); + d.add(ZT_REMOTE_TRACE_FIELD__FILTER_BASE_RULE_LOG,(const char *)primaryRuleSetLog.data(),(int)primaryRuleSetLog.sizeBytes()); + if (matchingCapabilityRuleSetLog) + d.add(ZT_REMOTE_TRACE_FIELD__FILTER_CAP_RULE_LOG,(const char *)matchingCapabilityRuleSetLog->data(),(int)matchingCapabilityRuleSetLog->sizeBytes()); + if (matchingCapability) + d.add(ZT_REMOTE_TRACE_FIELD__FILTER_CAP_ID,(uint64_t)matchingCapability->id()); + d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen); + if (frameLen > 0) + d.add(ZT_REMOTE_TRACE_FIELD__FRAME_DATA,(const char *)frameData,(frameLen > 256) ? (int)256 : (int)frameLen); + _send(tPtr,d,network.id()); } void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason) diff --git a/node/Trace.hpp b/node/Trace.hpp index dae67e28..7fe48cdd 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -93,7 +93,7 @@ public: } inline const uint8_t *data() const { return _l; } - inline unsigned int sizeBytes() const { return (unsigned int)sizeof(_l); } + inline unsigned int sizeBytes() const { return (ZT_MAX_NETWORK_RULES / 2); } private: uint8_t _l[ZT_MAX_NETWORK_RULES / 2]; @@ -115,7 +115,7 @@ public: void outgoingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason); void incomingNetworkAccessDenied(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested); - void incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac); + void incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac,const char *reason); void networkConfigRequestSent(void *const tPtr,const Network &network,const Address &controller); void networkFilter( -- cgit v1.2.3 From 3a1ec07db0d30415a21ab05be9898210d7cc70ef Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Jul 2017 10:43:28 -0700 Subject: Remove some exception copypasta. --- node/IncomingPacket.cpp | 1706 +++++++++++++++++++++++------------------------ 1 file changed, 830 insertions(+), 876 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 94c73f81..9489b16e 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -129,1106 +129,1060 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB]; - const uint64_t inRePacketId = at(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID); - const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE]; - uint64_t networkId = 0; - - /* Security note: we do not gate doERROR() with expectingReplyTo() to - * avoid having to log every outgoing packet ID. Instead we put the - * logic to determine whether we should consider an ERROR in each - * error handler. In most cases these are only trusted in specific - * circumstances. */ - - switch(errorCode) { - - case Packet::ERROR_OBJ_NOT_FOUND: - // Object not found, currently only meaningful from network controllers. - if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { - const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->controller() == peer->address())) - network->setNotFound(); - } - break; - - case Packet::ERROR_UNSUPPORTED_OPERATION: - // This can be sent in response to any operation, though right now we only - // consider it meaningful from network controllers. This would indicate - // that the queried node does not support acting as a controller. - if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { - const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); - if ((network)&&(network->controller() == peer->address())) - network->setNotFound(); - } - break; - - case Packet::ERROR_IDENTITY_COLLISION: - // FIXME: for federation this will need a payload with a signature or something. - if (RR->topology->isUpstream(peer->identity())) - RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); - break; - - case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { - // Peers can send this in response to frames if they do not have a recent enough COM from us - networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); - const SharedPtr network(RR->node->network(networkId)); - const uint64_t now = RR->node->now(); - if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) - network->pushCredentialsNow(tPtr,peer->address(),now); - } break; + const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB]; + const uint64_t inRePacketId = at(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID); + const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE]; + uint64_t networkId = 0; + + /* Security note: we do not gate doERROR() with expectingReplyTo() to + * avoid having to log every outgoing packet ID. Instead we put the + * logic to determine whether we should consider an ERROR in each + * error handler. In most cases these are only trusted in specific + * circumstances. */ + + switch(errorCode) { + + case Packet::ERROR_OBJ_NOT_FOUND: + // Object not found, currently only meaningful from network controllers. + if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + if ((network)&&(network->controller() == peer->address())) + network->setNotFound(); + } + break; - case Packet::ERROR_NETWORK_ACCESS_DENIED_: { - // Network controller: network access denied. + case Packet::ERROR_UNSUPPORTED_OPERATION: + // This can be sent in response to any operation, though right now we only + // consider it meaningful from network controllers. This would indicate + // that the queried node does not support acting as a controller. + if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) { const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); if ((network)&&(network->controller() == peer->address())) - network->setAccessDenied(); - } break; + network->setNotFound(); + } + break; + + case Packet::ERROR_IDENTITY_COLLISION: + // FIXME: for federation this will need a payload with a signature or something. + if (RR->topology->isUpstream(peer->identity())) + RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION); + break; + + case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: { + // Peers can send this in response to frames if they do not have a recent enough COM from us + networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); + const SharedPtr network(RR->node->network(networkId)); + const uint64_t now = RR->node->now(); + if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) + network->pushCredentialsNow(tPtr,peer->address(),now); + } break; + + case Packet::ERROR_NETWORK_ACCESS_DENIED_: { + // Network controller: network access denied. + const SharedPtr network(RR->node->network(at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD))); + if ((network)&&(network->controller() == peer->address())) + network->setAccessDenied(); + } break; + + case Packet::ERROR_UNWANTED_MULTICAST: { + // Members of networks can use this error to indicate that they no longer + // want to receive multicasts on a given channel. + networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); + const SharedPtr network(RR->node->network(networkId)); + if ((network)&&(network->gate(tPtr,peer))) { + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); + RR->mc->remove(network->id(),mg,peer->address()); + } + } break; - case Packet::ERROR_UNWANTED_MULTICAST: { - // Members of networks can use this error to indicate that they no longer - // want to receive multicasts on a given channel. - networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); - const SharedPtr network(RR->node->network(networkId)); - if ((network)&&(network->gate(tPtr,peer))) { - const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14)); - RR->mc->remove(network->id(),mg,peer->address()); - } - } break; + default: break; + } - default: break; - } + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,networkId); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,networkId); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_ERROR,"unexpected exception"); - } return true; } bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated) { - try { - const uint64_t now = RR->node->now(); - - const uint64_t pid = packetId(); - const Address fromAddress(source()); - const unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION]; - const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION]; - const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION]; - const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO_IDX_REVISION); - const uint64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); - Identity id; - unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); - - if (protoVersion < ZT_PROTO_VERSION_MIN) { - RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"protocol version too old"); - return true; - } - if (fromAddress != id.address()) { - RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"identity/address mismatch"); - return true; - } + const uint64_t now = RR->node->now(); - SharedPtr peer(RR->topology->getPeer(tPtr,id.address())); - if (peer) { - // We already have an identity with this address -- check for collisions - if (!alreadyAuthenticated) { - if (peer->identity() != id) { - // Identity is different from the one we already have -- address collision - - // Check rate limits - if (!RR->node->rateGateIdentityVerification(now,_path->address())) - return true; - - uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; - if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { - if (dearmor(key)) { // ensure packet is authentic, otherwise drop - RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"address collision"); - Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((uint8_t)Packet::VERB_HELLO); - outp.append((uint64_t)pid); - outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION); - outp.armor(key,true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - } else { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); - } - } else { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); - } + const uint64_t pid = packetId(); + const Address fromAddress(source()); + const unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION]; + const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION]; + const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION]; + const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO_IDX_REVISION); + const uint64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); + Identity id; + unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); + + if (protoVersion < ZT_PROTO_VERSION_MIN) { + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"protocol version too old"); + return true; + } + if (fromAddress != id.address()) { + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"identity/address mismatch"); + return true; + } + + SharedPtr peer(RR->topology->getPeer(tPtr,id.address())); + if (peer) { + // We already have an identity with this address -- check for collisions + if (!alreadyAuthenticated) { + if (peer->identity() != id) { + // Identity is different from the one we already have -- address collision + // Check rate limits + if (!RR->node->rateGateIdentityVerification(now,_path->address())) return true; - } else { - // Identity is the same as the one we already have -- check packet integrity - if (!dearmor(peer->key())) { + uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]; + if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) { + if (dearmor(key)) { // ensure packet is authentic, otherwise drop + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"address collision"); + Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((uint8_t)Packet::VERB_HELLO); + outp.append((uint64_t)pid); + outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION); + outp.armor(key,true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + } else { RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); - return true; } - - // Continue at // VALID + } else { + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); } - } // else if alreadyAuthenticated then continue at // VALID - } else { - // We don't already have an identity with this address -- validate and learn it - - // Sanity check: this basically can't happen - if (alreadyAuthenticated) { - RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"illegal alreadyAuthenticated state"); - return true; - } - // Check rate limits - if (!RR->node->rateGateIdentityVerification(now,_path->address())) { - RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"rate limit exceeded"); return true; - } + } else { + // Identity is the same as the one we already have -- check packet integrity - // Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap) - SharedPtr newPeer(new Peer(RR,RR->identity,id)); - if (!dearmor(newPeer->key())) { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); - return true; - } + if (!dearmor(peer->key())) { + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); + return true; + } - // Check that identity's address is valid as per the derivation function - if (!id.locallyValidate()) { - RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"invalid identity"); - return true; + // Continue at // VALID } + } // else if alreadyAuthenticated then continue at // VALID + } else { + // We don't already have an identity with this address -- validate and learn it - peer = RR->topology->addPeer(tPtr,newPeer); - - // Continue at // VALID + // Sanity check: this basically can't happen + if (alreadyAuthenticated) { + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"illegal alreadyAuthenticated state"); + return true; } - // VALID -- if we made it here, packet passed identity and authenticity checks! + // Check rate limits + if (!RR->node->rateGateIdentityVerification(now,_path->address())) { + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"rate limit exceeded"); + return true; + } - // Get external surface address if present (was not in old versions) - InetAddress externalSurfaceAddress; - if (ptr < size()) { - ptr += externalSurfaceAddress.deserialize(*this,ptr); - if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(tPtr,id.address(),_path->localSocket(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + // Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap) + SharedPtr newPeer(new Peer(RR,RR->identity,id)); + if (!dearmor(newPeer->key())) { + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); + return true; } - // Get primary planet world ID and world timestamp if present - uint64_t planetWorldId = 0; - uint64_t planetWorldTimestamp = 0; - if ((ptr + 16) <= size()) { - planetWorldId = at(ptr); ptr += 8; - planetWorldTimestamp = at(ptr); ptr += 8; + // Check that identity's address is valid as per the derivation function + if (!id.locallyValidate()) { + RR->t->incomingPacketDroppedHELLO(tPtr,_path,pid,fromAddress,"invalid identity"); + return true; } - std::vector< std::pair > moonIdsAndTimestamps; - if (ptr < size()) { - // Remainder of packet, if present, is encrypted - cryptField(peer->key(),ptr,size() - ptr); + peer = RR->topology->addPeer(tPtr,newPeer); - // Get moon IDs and timestamps if present - if ((ptr + 2) <= size()) { - const unsigned int numMoons = at(ptr); ptr += 2; - for(unsigned int i=0;i(at(ptr),at(ptr + 8))); - ptr += 16; - } - } + // Continue at // VALID + } - // Handle COR if present (older versions don't send this) - if ((ptr + 2) <= size()) { - if (at(ptr) > 0) { - CertificateOfRepresentation cor; - ptr += 2; - ptr += cor.deserialize(*this,ptr); - } else ptr += 2; - } - } + // VALID -- if we made it here, packet passed identity and authenticity checks! - // Send OK(HELLO) with an echo of the packet's timestamp and some of the same - // information about us: version, sent-to address, etc. + // Get external surface address if present (was not in old versions) + InetAddress externalSurfaceAddress; + if (ptr < size()) { + ptr += externalSurfaceAddress.deserialize(*this,ptr); + if ((externalSurfaceAddress)&&(hops() == 0)) + RR->sa->iam(tPtr,id.address(),_path->localSocket(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now); + } - Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_HELLO); - outp.append((uint64_t)pid); - outp.append((uint64_t)timestamp); - outp.append((unsigned char)ZT_PROTO_VERSION); - outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR); - outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR); - outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); + // Get primary planet world ID and world timestamp if present + uint64_t planetWorldId = 0; + uint64_t planetWorldTimestamp = 0; + if ((ptr + 16) <= size()) { + planetWorldId = at(ptr); ptr += 8; + planetWorldTimestamp = at(ptr); ptr += 8; + } - if (protoVersion >= 5) { - _path->address().serialize(outp); - } else { - /* LEGACY COMPATIBILITY HACK: - * - * For a while now (since 1.0.3), ZeroTier has recognized changes in - * its network environment empirically by examining its external network - * address as reported by trusted peers. In versions prior to 1.1.0 - * (protocol version < 5), they did this by saving a snapshot of this - * information (in SelfAwareness.hpp) keyed by reporting device ID and - * address type. - * - * This causes problems when clustering is combined with symmetric NAT. - * Symmetric NAT remaps ports, so different endpoints in a cluster will - * report back different exterior addresses. Since the old code keys - * this by device ID and not sending physical address and compares the - * entire address including port, it constantly thinks its external - * surface is changing and resets connections when talking to a cluster. - * - * In new code we key by sending physical address and device and we also - * take the more conservative position of only interpreting changes in - * IP address (neglecting port) as a change in network topology that - * necessitates a reset. But we can make older clients work here by - * nulling out the port field. Since this info is only used for empirical - * detection of link changes, it doesn't break anything else. - */ - InetAddress tmpa(_path->address()); - tmpa.setPort(0); - tmpa.serialize(outp); + std::vector< std::pair > moonIdsAndTimestamps; + if (ptr < size()) { + // Remainder of packet, if present, is encrypted + cryptField(peer->key(),ptr,size() - ptr); + + // Get moon IDs and timestamps if present + if ((ptr + 2) <= size()) { + const unsigned int numMoons = at(ptr); ptr += 2; + for(unsigned int i=0;i(at(ptr),at(ptr + 8))); + ptr += 16; + } } - const unsigned int worldUpdateSizeAt = outp.size(); - outp.addSize(2); // make room for 16-bit size field - if ((planetWorldId)&&(RR->topology->planetWorldTimestamp() > planetWorldTimestamp)&&(planetWorldId == RR->topology->planetWorldId())) { - RR->topology->planet().serialize(outp,false); + // Handle COR if present (older versions don't send this) + if ((ptr + 2) <= size()) { + if (at(ptr) > 0) { + CertificateOfRepresentation cor; + ptr += 2; + ptr += cor.deserialize(*this,ptr); + } else ptr += 2; } - if (moonIdsAndTimestamps.size() > 0) { - std::vector moons(RR->topology->moons()); - for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { - for(std::vector< std::pair >::const_iterator i(moonIdsAndTimestamps.begin());i!=moonIdsAndTimestamps.end();++i) { - if (i->first == m->id()) { - if (m->timestamp() > i->second) - m->serialize(outp,false); - break; - } + } + + // Send OK(HELLO) with an echo of the packet's timestamp and some of the same + // information about us: version, sent-to address, etc. + + Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_HELLO); + outp.append((uint64_t)pid); + outp.append((uint64_t)timestamp); + outp.append((unsigned char)ZT_PROTO_VERSION); + outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR); + outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR); + outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION); + + if (protoVersion >= 5) { + _path->address().serialize(outp); + } else { + /* LEGACY COMPATIBILITY HACK: + * + * For a while now (since 1.0.3), ZeroTier has recognized changes in + * its network environment empirically by examining its external network + * address as reported by trusted peers. In versions prior to 1.1.0 + * (protocol version < 5), they did this by saving a snapshot of this + * information (in SelfAwareness.hpp) keyed by reporting device ID and + * address type. + * + * This causes problems when clustering is combined with symmetric NAT. + * Symmetric NAT remaps ports, so different endpoints in a cluster will + * report back different exterior addresses. Since the old code keys + * this by device ID and not sending physical address and compares the + * entire address including port, it constantly thinks its external + * surface is changing and resets connections when talking to a cluster. + * + * In new code we key by sending physical address and device and we also + * take the more conservative position of only interpreting changes in + * IP address (neglecting port) as a change in network topology that + * necessitates a reset. But we can make older clients work here by + * nulling out the port field. Since this info is only used for empirical + * detection of link changes, it doesn't break anything else. + */ + InetAddress tmpa(_path->address()); + tmpa.setPort(0); + tmpa.serialize(outp); + } + + const unsigned int worldUpdateSizeAt = outp.size(); + outp.addSize(2); // make room for 16-bit size field + if ((planetWorldId)&&(RR->topology->planetWorldTimestamp() > planetWorldTimestamp)&&(planetWorldId == RR->topology->planetWorldId())) { + RR->topology->planet().serialize(outp,false); + } + if (moonIdsAndTimestamps.size() > 0) { + std::vector moons(RR->topology->moons()); + for(std::vector::const_iterator m(moons.begin());m!=moons.end();++m) { + for(std::vector< std::pair >::const_iterator i(moonIdsAndTimestamps.begin());i!=moonIdsAndTimestamps.end();++i) { + if (i->first == m->id()) { + if (m->timestamp() > i->second) + m->serialize(outp,false); + break; } } } - outp.setAt(worldUpdateSizeAt,(uint16_t)(outp.size() - (worldUpdateSizeAt + 2))); + } + outp.setAt(worldUpdateSizeAt,(uint16_t)(outp.size() - (worldUpdateSizeAt + 2))); - const unsigned int corSizeAt = outp.size(); - outp.addSize(2); - RR->topology->appendCertificateOfRepresentation(outp); - outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); + const unsigned int corSizeAt = outp.size(); + outp.addSize(2); + RR->topology->appendCertificateOfRepresentation(outp); + outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),now); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),now); + + peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version + peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false,0); - peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version - peer->received(tPtr,_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_HELLO,"unexpected exception"); - } return true; } bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; - const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); - uint64_t networkId = 0; - - if (!RR->node->expectingReplyTo(inRePacketId)) - return true; - - switch(inReVerb) { + const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB]; + const uint64_t inRePacketId = at(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID); + uint64_t networkId = 0; - case Packet::VERB_HELLO: { - const uint64_t latency = RR->node->now() - at(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP); - if (latency > ZT_HELLO_MAX_ALLOWABLE_LATENCY) - return true; + if (!RR->node->expectingReplyTo(inRePacketId)) + return true; - const unsigned int vProto = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION]; - const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION]; - const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION]; - const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION); - if (vProto < ZT_PROTO_VERSION_MIN) - return true; + switch(inReVerb) { - InetAddress externalSurfaceAddress; - unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2; - - // Get reported external surface address if present - if (ptr < size()) - ptr += externalSurfaceAddress.deserialize(*this,ptr); - - // Handle planet or moon updates if present - if ((ptr + 2) <= size()) { - const unsigned int worldsLen = at(ptr); ptr += 2; - if (RR->topology->shouldAcceptWorldUpdateFrom(peer->address())) { - const unsigned int endOfWorlds = ptr + worldsLen; - while (ptr < endOfWorlds) { - World w; - ptr += w.deserialize(*this,ptr); - RR->topology->addWorld(tPtr,w,false); - } - } else { - ptr += worldsLen; - } - } + case Packet::VERB_HELLO: { + const uint64_t latency = RR->node->now() - at(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP); + if (latency > ZT_HELLO_MAX_ALLOWABLE_LATENCY) + return true; - // Handle certificate of representation if present - if ((ptr + 2) <= size()) { - if (at(ptr) > 0) { - CertificateOfRepresentation cor; - ptr += 2; - ptr += cor.deserialize(*this,ptr); - } else ptr += 2; - } + const unsigned int vProto = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION]; + const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION]; + const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION]; + const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION); + if (vProto < ZT_PROTO_VERSION_MIN) + return true; - if (!hops()) - peer->addDirectLatencyMeasurment((unsigned int)latency); - peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); + InetAddress externalSurfaceAddress; + unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2; - if ((externalSurfaceAddress)&&(hops() == 0)) - RR->sa->iam(tPtr,peer->address(),_path->localSocket(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); - } break; + // Get reported external surface address if present + if (ptr < size()) + ptr += externalSurfaceAddress.deserialize(*this,ptr); - case Packet::VERB_WHOIS: - if (RR->topology->isUpstream(peer->identity())) { - const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY); - RR->sw->doAnythingWaitingForPeer(tPtr,RR->topology->addPeer(tPtr,SharedPtr(new Peer(RR,RR->identity,id)))); + // Handle planet or moon updates if present + if ((ptr + 2) <= size()) { + const unsigned int worldsLen = at(ptr); ptr += 2; + if (RR->topology->shouldAcceptWorldUpdateFrom(peer->address())) { + const unsigned int endOfWorlds = ptr + worldsLen; + while (ptr < endOfWorlds) { + World w; + ptr += w.deserialize(*this,ptr); + RR->topology->addWorld(tPtr,w,false); + } + } else { + ptr += worldsLen; } - break; + } - case Packet::VERB_NETWORK_CONFIG_REQUEST: { - networkId = at(ZT_PROTO_VERB_OK_IDX_PAYLOAD); - const SharedPtr network(RR->node->network(networkId)); - if (network) - network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); - } break; + // Handle certificate of representation if present + if ((ptr + 2) <= size()) { + if (at(ptr) > 0) { + CertificateOfRepresentation cor; + ptr += 2; + ptr += cor.deserialize(*this,ptr); + } else ptr += 2; + } - case Packet::VERB_MULTICAST_GATHER: { - networkId = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); - const SharedPtr network(RR->node->network(networkId)); - if (network) { - const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); - const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); - RR->mc->addMultiple(tPtr,RR->node->now(),networkId,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); - } - } break; + if (!hops()) + peer->addDirectLatencyMeasurment((unsigned int)latency); + peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); - case Packet::VERB_MULTICAST_FRAME: { - const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS]; - networkId = at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID); - const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI)); + if ((externalSurfaceAddress)&&(hops() == 0)) + RR->sa->iam(tPtr,peer->address(),_path->localSocket(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now()); + } break; - const SharedPtr network(RR->node->network(networkId)); - if (network) { - unsigned int offset = 0; + case Packet::VERB_WHOIS: + if (RR->topology->isUpstream(peer->identity())) { + const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY); + RR->sw->doAnythingWaitingForPeer(tPtr,RR->topology->addPeer(tPtr,SharedPtr(new Peer(RR,RR->identity,id)))); + } + break; + + case Packet::VERB_NETWORK_CONFIG_REQUEST: { + networkId = at(ZT_PROTO_VERB_OK_IDX_PAYLOAD); + const SharedPtr network(RR->node->network(networkId)); + if (network) + network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PROTO_VERB_OK_IDX_PAYLOAD); + } break; + + case Packet::VERB_MULTICAST_GATHER: { + networkId = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(networkId)); + if (network) { + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI)); + const unsigned int count = at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4); + RR->mc->addMultiple(tPtr,RR->node->now(),networkId,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS)); + } + } break; - if ((flags & 0x01) != 0) { // deprecated but still used by older peers - CertificateOfMembership com; - offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); - if (com) - network->addCredential(tPtr,com); - } + case Packet::VERB_MULTICAST_FRAME: { + const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS]; + networkId = at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID); + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI)); - if ((flags & 0x02) != 0) { - // OK(MULTICAST_FRAME) includes implicit gather results - offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; - unsigned int totalKnown = at(offset); offset += 4; - unsigned int count = at(offset); offset += 2; - RR->mc->addMultiple(tPtr,RR->node->now(),networkId,mg,field(offset,count * 5),count,totalKnown); - } + const SharedPtr network(RR->node->network(networkId)); + if (network) { + unsigned int offset = 0; + + if ((flags & 0x01) != 0) { // deprecated but still used by older peers + CertificateOfMembership com; + offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS); + if (com) + network->addCredential(tPtr,com); } - } break; - default: break; - } + if ((flags & 0x02) != 0) { + // OK(MULTICAST_FRAME) includes implicit gather results + offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS; + unsigned int totalKnown = at(offset); offset += 4; + unsigned int count = at(offset); offset += 2; + RR->mc->addMultiple(tPtr,RR->node->now(),networkId,mg,field(offset,count * 5),count,totalKnown); + } + } + } break; - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false,networkId); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_OK,"unexpected exception"); + default: break; } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false,networkId); + return true; } bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) - return true; - - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_WHOIS); - outp.append(packetId()); + if ((!RR->topology->amRoot())&&(!peer->rateGateInboundWhoisRequest(RR->node->now()))) + return true; - unsigned int count = 0; - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD; - while ((ptr + ZT_ADDRESS_LENGTH) <= size()) { - const Address addr(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - ptr += ZT_ADDRESS_LENGTH; + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_WHOIS); + outp.append(packetId()); - const Identity id(RR->topology->getIdentity(tPtr,addr)); - if (id) { - id.serialize(outp,false); - ++count; - } else { - // Request unknown WHOIS from upstream from us (if we have one) - RR->sw->requestWhois(tPtr,addr); - } - } + unsigned int count = 0; + unsigned int ptr = ZT_PACKET_IDX_PAYLOAD; + while ((ptr + ZT_ADDRESS_LENGTH) <= size()) { + const Address addr(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + ptr += ZT_ADDRESS_LENGTH; - if (count > 0) { - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + const Identity id(RR->topology->getIdentity(tPtr,addr)); + if (id) { + id.serialize(outp,false); + ++count; + } else { + // Request unknown WHOIS from upstream from us (if we have one) + RR->sw->requestWhois(tPtr,addr); } + } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_WHOIS,"unexpected exception"); + if (count > 0) { + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,0); + return true; } bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - if (RR->topology->isUpstream(peer->identity())) { - const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); - const SharedPtr rendezvousWith(RR->topology->getPeer(tPtr,with)); - if (rendezvousWith) { - const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); - const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; - if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { - const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); - if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localSocket(),atAddr)) { - const uint64_t junk = RR->node->prng(); - RR->node->putPacket(tPtr,_path->localSocket(),atAddr,&junk,4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls - rendezvousWith->attemptToContactAt(tPtr,_path->localSocket(),atAddr,RR->node->now(),false,0); - } + if (RR->topology->isUpstream(peer->identity())) { + const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); + const SharedPtr rendezvousWith(RR->topology->getPeer(tPtr,with)); + if (rendezvousWith) { + const unsigned int port = at(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT); + const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN]; + if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) { + const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port); + if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,_path->localSocket(),atAddr)) { + const uint64_t junk = RR->node->prng(); + RR->node->putPacket(tPtr,_path->localSocket(),atAddr,&junk,4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls + rendezvousWith->attemptToContactAt(tPtr,_path->localSocket(),atAddr,RR->node->now(),false,0); } } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_RENDEZVOUS,"unexpected exception"); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,0); + return true; } bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); - const SharedPtr network(RR->node->network(nwid)); - bool trustEstablished = false; - if (network) { - if (network->gate(tPtr,peer)) { - trustEstablished = true; - if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { - const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); - const MAC sourceMac(peer->address(),nwid); - const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; - if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) - RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); - } - } else { - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_FRAME,true); + const uint64_t nwid = at(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(nwid)); + bool trustEstablished = false; + if (network) { + if (network->gate(tPtr,peer)) { + trustEstablished = true; + if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) { + const unsigned int etherType = at(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE); + const MAC sourceMac(peer->address(),nwid); + const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; + const uint8_t *const frameData = reinterpret_cast(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD; + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) + RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen); } } else { _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_FRAME,true); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished,nwid); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_FRAME,"unexpected exception"); + } else { + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished,nwid); + return true; } bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t nwid = at(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); - const SharedPtr network(RR->node->network(nwid)); - if (network) { - const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; - - unsigned int comLen = 0; - if ((flags & 0x01) != 0) { // inline COM with EXT_FRAME is deprecated but still used with old peers - CertificateOfMembership com; - comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM); - if (com) - network->addCredential(tPtr,com); - } + const uint64_t nwid = at(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID); + const SharedPtr network(RR->node->network(nwid)); + if (network) { + const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS]; + + unsigned int comLen = 0; + if ((flags & 0x01) != 0) { // inline COM with EXT_FRAME is deprecated but still used with old peers + CertificateOfMembership com; + comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM); + if (com) + network->addCredential(tPtr,com); + } - if (!network->gate(tPtr,peer)) { - RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,true); - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); - return true; - } + if (!network->gate(tPtr,peer)) { + RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,true); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); + return true; + } - if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { - const unsigned int etherType = at(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE); - const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO); - const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM); - const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); - const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); + if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) { + const unsigned int etherType = at(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE); + const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO); + const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM); + const unsigned int frameLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD); + const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen); - if ((!from)||(from == network->mac())) { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay - return true; - } + if ((!from)||(from == network->mac())) { + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + return true; + } - switch (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { - case 1: - if (from != MAC(peer->address(),nwid)) { - if (network->config().permitsBridging(peer->address())) { - network->learnBridgeRoute(from,peer->address()); - } else { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (remote)"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay - return true; - } - } else if (to != network->mac()) { - if (to.isMulticast()) { - if (network->config().multicastLimit == 0) { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"multicast disabled"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay - return true; - } - } else if (!network->config().permitsBridging(RR->identity.address())) { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (local)"); + switch (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to,frameData,frameLen,etherType,0)) { + case 1: + if (from != MAC(peer->address(),nwid)) { + if (network->config().permitsBridging(peer->address())) { + network->learnBridgeRoute(from,peer->address()); + } else { + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (remote)"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + return true; + } + } else if (to != network->mac()) { + if (to.isMulticast()) { + if (network->config().multicastLimit == 0) { + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"multicast disabled"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } + } else if (!network->config().permitsBridging(RR->identity.address())) { + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (local)"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + return true; } - // fall through -- 2 means accept regardless of bridging checks or other restrictions - case 2: - RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); - break; - } - } - - if ((flags & 0x10) != 0) { // ACK requested - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((uint8_t)Packet::VERB_EXT_FRAME); - outp.append((uint64_t)packetId()); - outp.append((uint64_t)nwid); - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + } + // fall through -- 2 means accept regardless of bridging checks or other restrictions + case 2: + RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen); + break; } + } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); - } else { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); + if ((flags & 0x10) != 0) { // ACK requested + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((uint8_t)Packet::VERB_EXT_FRAME); + outp.append((uint64_t)packetId()); + outp.append((uint64_t)nwid); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_EXT_FRAME,"unexpected exception"); + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); + } else { + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid); } + return true; } bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - if (!peer->rateGateEchoRequest(RR->node->now())) - return true; + if (!peer->rateGateEchoRequest(RR->node->now())) + return true; - const uint64_t pid = packetId(); - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_ECHO); - outp.append((uint64_t)pid); - if (size() > ZT_PACKET_IDX_PAYLOAD) - outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + const uint64_t pid = packetId(); + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_ECHO); + outp.append((uint64_t)pid); + if (size() > ZT_PACKET_IDX_PAYLOAD) + outp.append(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + + peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false,0); - peer->received(tPtr,_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_ECHO,"unexpected exception"); - } return true; } bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t now = RR->node->now(); + const uint64_t now = RR->node->now(); - uint64_t authOnNetwork[256]; // cache for approved network IDs - unsigned int authOnNetworkCount = 0; - SharedPtr network; - bool trustEstablished = false; + uint64_t authOnNetwork[256]; // cache for approved network IDs + unsigned int authOnNetworkCount = 0; + SharedPtr network; + bool trustEstablished = false; - // Iterate through 18-byte network,MAC,ADI tuples - for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr(ptr); + // Iterate through 18-byte network,MAC,ADI tuples + for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr(ptr); - bool auth = false; - for(unsigned int i=0;iid() != nwid)) - network = RR->node->network(nwid); - const bool authOnNet = ((network)&&(network->gate(tPtr,peer))); - if (!authOnNet) - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - trustEstablished |= authOnNet; - if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { - auth = true; - if (authOnNetworkCount < 256) // sanity check, packets can't really be this big - authOnNetwork[authOnNetworkCount++] = nwid; - } + bool auth = false; + for(unsigned int i=0;i(ptr + 14)); - RR->mc->add(tPtr,now,nwid,group,peer->address()); + } + if (!auth) { + if ((!network)||(network->id() != nwid)) + network = RR->node->network(nwid); + const bool authOnNet = ((network)&&(network->gate(tPtr,peer))); + if (!authOnNet) + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + trustEstablished |= authOnNet; + if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) { + auth = true; + if (authOnNetworkCount < 256) // sanity check, packets can't really be this big + authOnNetwork[authOnNetworkCount++] = nwid; } } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_LIKE,"unexpected exception"); + if (auth) { + const MulticastGroup group(MAC(field(ptr + 8,6),6),at(ptr + 14)); + RR->mc->add(tPtr,now,nwid,group,peer->address()); + } } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); + return true; } bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - if (!peer->rateGateCredentialsReceived(RR->node->now())) - return true; + if (!peer->rateGateCredentialsReceived(RR->node->now())) + return true; - CertificateOfMembership com; - Capability cap; - Tag tag; - Revocation revocation; - CertificateOfOwnership coo; - bool trustEstablished = false; - SharedPtr network; - - unsigned int p = ZT_PACKET_IDX_PAYLOAD; - while ((p < size())&&((*this)[p] != 0)) { - p += com.deserialize(*this,p); - if (com) { - network = RR->node->network(com.networkId()); - if (network) { - switch (network->addCredential(tPtr,com)) { - case Membership::ADD_REJECTED: - break; - case Membership::ADD_ACCEPTED_NEW: - case Membership::ADD_ACCEPTED_REDUNDANT: - trustEstablished = true; - break; - case Membership::ADD_DEFERRED_FOR_WHOIS: - return false; - } - } else RR->mc->addCredential(tPtr,com,false); - } + CertificateOfMembership com; + Capability cap; + Tag tag; + Revocation revocation; + CertificateOfOwnership coo; + bool trustEstablished = false; + SharedPtr network; + + unsigned int p = ZT_PACKET_IDX_PAYLOAD; + while ((p < size())&&((*this)[p] != 0)) { + p += com.deserialize(*this,p); + if (com) { + network = RR->node->network(com.networkId()); + if (network) { + switch (network->addCredential(tPtr,com)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; + } + } else RR->mc->addCredential(tPtr,com,false); } - ++p; // skip trailing 0 after COMs if present - - if (p < size()) { // older ZeroTier versions do not send capabilities, tags, or revocations - const unsigned int numCapabilities = at(p); p += 2; - for(unsigned int i=0;iid() != cap.networkId())) - network = RR->node->network(cap.networkId()); - if (network) { - switch (network->addCredential(tPtr,cap)) { - case Membership::ADD_REJECTED: - break; - case Membership::ADD_ACCEPTED_NEW: - case Membership::ADD_ACCEPTED_REDUNDANT: - trustEstablished = true; - break; - case Membership::ADD_DEFERRED_FOR_WHOIS: - return false; - } + } + ++p; // skip trailing 0 after COMs if present + + if (p < size()) { // older ZeroTier versions do not send capabilities, tags, or revocations + const unsigned int numCapabilities = at(p); p += 2; + for(unsigned int i=0;iid() != cap.networkId())) + network = RR->node->network(cap.networkId()); + if (network) { + switch (network->addCredential(tPtr,cap)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } + } - if (p >= size()) return true; - - const unsigned int numTags = at(p); p += 2; - for(unsigned int i=0;iid() != tag.networkId())) - network = RR->node->network(tag.networkId()); - if (network) { - switch (network->addCredential(tPtr,tag)) { - case Membership::ADD_REJECTED: - break; - case Membership::ADD_ACCEPTED_NEW: - case Membership::ADD_ACCEPTED_REDUNDANT: - trustEstablished = true; - break; - case Membership::ADD_DEFERRED_FOR_WHOIS: - return false; - } + if (p >= size()) return true; + + const unsigned int numTags = at(p); p += 2; + for(unsigned int i=0;iid() != tag.networkId())) + network = RR->node->network(tag.networkId()); + if (network) { + switch (network->addCredential(tPtr,tag)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } + } - if (p >= size()) return true; - - const unsigned int numRevocations = at(p); p += 2; - for(unsigned int i=0;iid() != revocation.networkId())) - network = RR->node->network(revocation.networkId()); - if (network) { - switch(network->addCredential(tPtr,peer->address(),revocation)) { - case Membership::ADD_REJECTED: - break; - case Membership::ADD_ACCEPTED_NEW: - case Membership::ADD_ACCEPTED_REDUNDANT: - trustEstablished = true; - break; - case Membership::ADD_DEFERRED_FOR_WHOIS: - return false; - } + if (p >= size()) return true; + + const unsigned int numRevocations = at(p); p += 2; + for(unsigned int i=0;iid() != revocation.networkId())) + network = RR->node->network(revocation.networkId()); + if (network) { + switch(network->addCredential(tPtr,peer->address(),revocation)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } + } - if (p >= size()) return true; - - const unsigned int numCoos = at(p); p += 2; - for(unsigned int i=0;iid() != coo.networkId())) - network = RR->node->network(coo.networkId()); - if (network) { - switch(network->addCredential(tPtr,coo)) { - case Membership::ADD_REJECTED: - break; - case Membership::ADD_ACCEPTED_NEW: - case Membership::ADD_ACCEPTED_REDUNDANT: - trustEstablished = true; - break; - case Membership::ADD_DEFERRED_FOR_WHOIS: - return false; - } + if (p >= size()) return true; + + const unsigned int numCoos = at(p); p += 2; + for(unsigned int i=0;iid() != coo.networkId())) + network = RR->node->network(coo.networkId()); + if (network) { + switch(network->addCredential(tPtr,coo)) { + case Membership::ADD_REJECTED: + break; + case Membership::ADD_ACCEPTED_NEW: + case Membership::ADD_ACCEPTED_REDUNDANT: + trustEstablished = true; + break; + case Membership::ADD_DEFERRED_FOR_WHOIS: + return false; } } } - - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_NETWORK_CREDENTIALS,"unexpected exception"); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0); + return true; } bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); - const unsigned int hopCount = hops(); - const uint64_t requestPacketId = packetId(); - - if (RR->localNetworkController) { - const unsigned int metaDataLength = (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN <= size()) ? at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN) : 0; - const char *metaDataBytes = (metaDataLength != 0) ? (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength) : (const char *)0; - const Dictionary metaData(metaDataBytes,metaDataLength); - RR->localNetworkController->request(nwid,(hopCount > 0) ? InetAddress() : _path->address(),requestPacketId,peer->identity(),metaData); - } else { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); - outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); - outp.append(requestPacketId); - outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); - outp.append(nwid); - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - } - - peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false,nwid); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_NETWORK_CONFIG_REQUEST,"unexpected exception"); + const uint64_t nwid = at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID); + const unsigned int hopCount = hops(); + const uint64_t requestPacketId = packetId(); + + if (RR->localNetworkController) { + const unsigned int metaDataLength = (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN <= size()) ? at(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN) : 0; + const char *metaDataBytes = (metaDataLength != 0) ? (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength) : (const char *)0; + const Dictionary metaData(metaDataBytes,metaDataLength); + RR->localNetworkController->request(nwid,(hopCount > 0) ? InetAddress() : _path->address(),requestPacketId,peer->identity(),metaData); + } else { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR); + outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); + outp.append(requestPacketId); + outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION); + outp.append(nwid); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } + + peer->received(tPtr,_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false,nwid); + return true; } bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); - if (network) { - const uint64_t configUpdateId = network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); - if (configUpdateId) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((uint8_t)Packet::VERB_ECHO); - outp.append((uint64_t)packetId()); - outp.append((uint64_t)network->id()); - outp.append((uint64_t)configUpdateId); - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - } + const SharedPtr network(RR->node->network(at(ZT_PACKET_IDX_PAYLOAD))); + if (network) { + const uint64_t configUpdateId = network->handleConfigChunk(tPtr,packetId(),source(),*this,ZT_PACKET_IDX_PAYLOAD); + if (configUpdateId) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((uint8_t)Packet::VERB_ECHO); + outp.append((uint64_t)packetId()); + outp.append((uint64_t)network->id()); + outp.append((uint64_t)configUpdateId); + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false,(network) ? network->id() : 0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_NETWORK_CONFIG,"unexpected exception"); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false,(network) ? network->id() : 0); + return true; } bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); - const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS]; - const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); - const unsigned int gatherLimit = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); - - const SharedPtr network(RR->node->network(nwid)); + const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID); + const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS]; + const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI)); + const unsigned int gatherLimit = at(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT); - if ((flags & 0x01) != 0) { - try { - CertificateOfMembership com; - com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); - if (com) { - if (network) - network->addCredential(tPtr,com); - else RR->mc->addCredential(tPtr,com,false); - } - } catch ( ... ) {} // discard invalid COMs - } + const SharedPtr network(RR->node->network(nwid)); - const bool trustEstablished = ((network)&&(network->gate(tPtr,peer))); - if (!trustEstablished) - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { - Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); - outp.append(packetId()); - outp.append(nwid); - mg.mac().appendTo(outp); - outp.append((uint32_t)mg.adi()); - const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); - if (gatheredLocally > 0) { - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + if ((flags & 0x01) != 0) { + try { + CertificateOfMembership com; + com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM); + if (com) { + if (network) + network->addCredential(tPtr,com); + else RR->mc->addCredential(tPtr,com,false); } - } + } catch ( ... ) {} // discard invalid COMs + } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished,nwid); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_GATHER,"unexpected exception"); + const bool trustEstablished = ((network)&&(network->gate(tPtr,peer))); + if (!trustEstablished) + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) { + Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER); + outp.append(packetId()); + outp.append(nwid); + mg.mac().appendTo(outp); + outp.append((uint32_t)mg.adi()); + const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit); + if (gatheredLocally > 0) { + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + } } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished,nwid); + return true; } bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID); - const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS]; - - const SharedPtr network(RR->node->network(nwid)); - if (network) { - // Offset -- size of optional fields added to position of later fields - unsigned int offset = 0; - - if ((flags & 0x01) != 0) { - // This is deprecated but may still be sent by old peers - CertificateOfMembership com; - offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); - if (com) - network->addCredential(tPtr,com); - } + const uint64_t nwid = at(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID); + const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS]; - if (!network->gate(tPtr,peer)) { - RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,true); - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); - return true; - } + const SharedPtr network(RR->node->network(nwid)); + if (network) { + // Offset -- size of optional fields added to position of later fields + unsigned int offset = 0; - unsigned int gatherLimit = 0; - if ((flags & 0x02) != 0) { - gatherLimit = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT); - offset += 4; - } + if ((flags & 0x01) != 0) { + // This is deprecated but may still be sent by old peers + CertificateOfMembership com; + offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM); + if (com) + network->addCredential(tPtr,com); + } - MAC from; - if ((flags & 0x04) != 0) { - from.setTo(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6),6); - offset += 6; - } else { - from.fromAddress(peer->address(),nwid); - } + if (!network->gate(tPtr,peer)) { + RR->t->incomingNetworkAccessDenied(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,true); + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); + return true; + } + + unsigned int gatherLimit = 0; + if ((flags & 0x02) != 0) { + gatherLimit = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT); + offset += 4; + } + + MAC from; + if ((flags & 0x04) != 0) { + from.setTo(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6),6); + offset += 6; + } else { + from.fromAddress(peer->address(),nwid); + } - const MulticastGroup to(MAC(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI)); - const unsigned int etherType = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE); - const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); + const MulticastGroup to(MAC(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI)); + const unsigned int etherType = at(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE); + const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME); - if (network->config().multicastLimit == 0) { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"multicast disabled"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); + if (network->config().multicastLimit == 0) { + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"multicast disabled"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); + return true; + } + + if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) { + if (!to.mac().isMulticast()) { + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay + return true; + } + if ((!from)||(from.isMulticast())||(from == network->mac())) { + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC"); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } - if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) { - if (!to.mac().isMulticast()) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay - return true; - } - if ((!from)||(from.isMulticast())||(from == network->mac())) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC"); + if (from != MAC(peer->address(),nwid)) { + if (network->config().permitsBridging(peer->address())) { + network->learnBridgeRoute(from,peer->address()); + } else { + RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"bridging not allowed (remote)"); peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay return true; } - - if (from != MAC(peer->address(),nwid)) { - if (network->config().permitsBridging(peer->address())) { - network->learnBridgeRoute(from,peer->address()); - } else { - RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"bridging not allowed (remote)"); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay - return true; - } - } - - const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); - if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) - RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); } - if (gatherLimit) { - Packet outp(source(),RR->identity.address(),Packet::VERB_OK); - outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME); - outp.append(packetId()); - outp.append(nwid); - to.mac().appendTo(outp); - outp.append((uint32_t)to.adi()); - outp.append((unsigned char)0x02); // flag 0x02 = contains gather results - if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) { - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); - _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); - } - } + const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen); + if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) + RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen); + } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); - } else { - _sendErrorNeedCredentials(RR,tPtr,peer,nwid); - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); + if (gatherLimit) { + Packet outp(source(),RR->identity.address(),Packet::VERB_OK); + outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME); + outp.append(packetId()); + outp.append(nwid); + to.mac().appendTo(outp); + outp.append((uint32_t)to.adi()); + outp.append((unsigned char)0x02); // flag 0x02 = contains gather results + if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) { + outp.armor(peer->key(),true,_path->nextOutgoingCounter()); + _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); + } } - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"unexpected exception"); + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); + } else { + _sendErrorNeedCredentials(RR,tPtr,peer,nwid); + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid); } + return true; } bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - const uint64_t now = RR->node->now(); + const uint64_t now = RR->node->now(); - // First, subject this to a rate limit - if (!peer->rateGatePushDirectPaths(now)) { - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); - return true; - } + // First, subject this to a rate limit + if (!peer->rateGatePushDirectPaths(now)) { + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); + return true; + } - // Second, limit addresses by scope and type - uint8_t countPerScope[ZT_INETADDRESS_MAX_SCOPE+1][2]; // [][0] is v4, [][1] is v6 - memset(countPerScope,0,sizeof(countPerScope)); - - unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); - unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2; - - while (count--) { // if ptr overflows Buffer will throw - // TODO: some flags are not yet implemented - - unsigned int flags = (*this)[ptr++]; - unsigned int extLen = at(ptr); ptr += 2; - ptr += extLen; // unused right now - unsigned int addrType = (*this)[ptr++]; - unsigned int addrLen = (*this)[ptr++]; - - switch(addrType) { - case 4: { - const InetAddress a(field(ptr,4),4,at(ptr + 4)); - if ( - ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget - (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known - (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path - { - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->redirect(tPtr,_path->localSocket(),a,now); - } else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { - peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); - } + // Second, limit addresses by scope and type + uint8_t countPerScope[ZT_INETADDRESS_MAX_SCOPE+1][2]; // [][0] is v4, [][1] is v6 + memset(countPerScope,0,sizeof(countPerScope)); + + unsigned int count = at(ZT_PACKET_IDX_PAYLOAD); + unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2; + + while (count--) { // if ptr overflows Buffer will throw + // TODO: some flags are not yet implemented + + unsigned int flags = (*this)[ptr++]; + unsigned int extLen = at(ptr); ptr += 2; + ptr += extLen; // unused right now + unsigned int addrType = (*this)[ptr++]; + unsigned int addrLen = (*this)[ptr++]; + + switch(addrType) { + case 4: { + const InetAddress a(field(ptr,4),4,at(ptr + 4)); + if ( + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path + { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { + peer->redirect(tPtr,_path->localSocket(),a,now); + } else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { + peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } - } break; - case 6: { - const InetAddress a(field(ptr,16),16,at(ptr + 16)); - if ( - ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget - (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known - (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path - { - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->redirect(tPtr,_path->localSocket(),a,now); - } else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { - peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); - } + } + } break; + case 6: { + const InetAddress a(field(ptr,16),16,at(ptr + 16)); + if ( + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path + { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { + peer->redirect(tPtr,_path->localSocket(),a,now); + } else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { + peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } - } break; - } - ptr += addrLen; + } + } break; } - - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_PUSH_DIRECT_PATHS,"unexpected exception"); + ptr += addrLen; } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0); + return true; } bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - try { - if (likely(size() >= (ZT_PACKET_IDX_PAYLOAD + 8))) { - ZT_UserMessage um; - um.origin = peer->address().toInt(); - um.typeId = at(ZT_PACKET_IDX_PAYLOAD); - um.data = reinterpret_cast(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD + 8); - um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); - RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); - } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_USER_MESSAGE,"unexpected exception"); + if (likely(size() >= (ZT_PACKET_IDX_PAYLOAD + 8))) { + ZT_UserMessage um; + um.origin = peer->address().toInt(); + um.typeId = at(ZT_PACKET_IDX_PAYLOAD); + um.data = reinterpret_cast(reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD + 8); + um.length = size() - (ZT_PACKET_IDX_PAYLOAD + 8); + RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast(&um)); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false,0); + return true; } bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { ZT_RemoteTrace rt; - try { - const char *ptr = reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD; - const char *const eof = reinterpret_cast(data()) + size(); - rt.origin = peer->address().toInt(); - rt.data = const_cast(ptr); // start of first string - while (ptr < eof) { - if (!*ptr) { // end of string - rt.len = (unsigned int)(ptr - rt.data); - if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) - RR->node->postEvent(tPtr,ZT_EVENT_REMOTE_TRACE,&rt); - rt.data = const_cast(++ptr); // start of next string, if any - } else { - ++ptr; - } + const char *ptr = reinterpret_cast(data()) + ZT_PACKET_IDX_PAYLOAD; + const char *const eof = reinterpret_cast(data()) + size(); + rt.origin = peer->address().toInt(); + rt.data = const_cast(ptr); // start of first string + while (ptr < eof) { + if (!*ptr) { // end of string + rt.len = (unsigned int)(ptr - rt.data); + if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) + RR->node->postEvent(tPtr,ZT_EVENT_REMOTE_TRACE,&rt); + rt.data = const_cast(++ptr); // start of next string, if any + } else { + ++ptr; } - peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0); - } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_REMOTE_TRACE,"unexpected exception"); } + + peer->received(tPtr,_path,hops(),packetId(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0); + return true; } -- cgit v1.2.3 From ab0806a036485979d60015a22a8e8831b68643a2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Jul 2017 13:48:39 -0700 Subject: Cleanup. --- node/IncomingPacket.cpp | 5 ++--- node/Packet.hpp | 6 ------ node/Switch.cpp | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 9489b16e..be3d082b 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -99,7 +99,6 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) default: // ignore unknown verbs, but if they pass auth check they are "received" peer->received(tPtr,_path,hops(),packetId(),v,0,Packet::VERB_NOP,false,0); return true; - case Packet::VERB_HELLO: return _doHELLO(RR,tPtr,true); case Packet::VERB_ERROR: return _doERROR(RR,tPtr,peer); case Packet::VERB_OK: return _doOK(RR,tPtr,peer); @@ -122,7 +121,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) return false; } } catch ( ... ) { - RR->t->incomingPacketInvalid(tPtr,_path,packetId(),sourceAddress,hops(),verb(),"unexpected exception in tryDecode() (outer)"); + RR->t->incomingPacketInvalid(tPtr,_path,packetId(),sourceAddress,hops(),verb(),"unexpected exception in tryDecode()"); return true; } } @@ -332,7 +331,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool } } - // Handle COR if present (older versions don't send this) + // Certificates of representation (if present) if ((ptr + 2) <= size()) { if (at(ptr) > 0) { CertificateOfRepresentation cor; diff --git a/node/Packet.hpp b/node/Packet.hpp index b8e69fa9..4192b4e3 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -1274,12 +1274,6 @@ public: /** * Encrypt/decrypt a separately armored portion of a packet * - * This currently uses Salsa20/12, but any message that uses this should - * incorporate a cipher selector to permit this to be changed later. To - * ensure that key stream is not reused, the key is slightly altered for - * this use case and the same initial 32 keystream bytes that are taken - * for MAC in ordinary armor() are also skipped here. - * * This is currently only used to mask portions of HELLO as an extra * security precation since most of that message is sent in the clear. * diff --git a/node/Switch.cpp b/node/Switch.cpp index eee49775..9c9daac9 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -60,7 +60,7 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre try { const uint64_t now = RR->node->now(); - SharedPtr path(RR->topology->getPath(localSocket,fromAddr)); + const SharedPtr path(RR->topology->getPath(localSocket,fromAddr)); path->received(now); if (len == 13) { -- cgit v1.2.3 From b9e1d53d7ac4e8d19520e3063b194ee01f550198 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Jul 2017 14:21:09 -0700 Subject: Minor cleanup. --- node/Array.hpp | 60 +++++++++++++++++++----------------- node/Buffer.hpp | 42 ++++++++++++------------- node/C25519.cpp | 5 --- node/C25519.hpp | 32 +++++-------------- node/Capability.hpp | 10 +++--- node/CertificateOfMembership.hpp | 8 ++--- node/CertificateOfOwnership.hpp | 4 +-- node/CertificateOfRepresentation.hpp | 4 +-- node/Constants.hpp | 9 ++++++ node/Hashtable.hpp | 8 ++--- node/Identity.cpp | 5 ++- node/Identity.hpp | 29 +++++++++-------- node/IncomingPacket.hpp | 2 +- node/InetAddress.hpp | 2 +- node/MAC.hpp | 2 +- node/MulticastGroup.hpp | 25 +++++++-------- node/Multicaster.hpp | 4 +-- node/Mutex.hpp | 22 +++---------- node/Network.hpp | 4 +-- node/NetworkConfig.hpp | 14 ++++----- node/Node.cpp | 2 +- node/Node.hpp | 4 +-- node/NonCopyable.hpp | 2 +- node/OutboundMulticast.hpp | 6 ++-- node/Packet.hpp | 9 ++---- node/Peer.cpp | 2 +- node/Peer.hpp | 4 +-- node/Poly1305.cpp | 2 -- node/Poly1305.hpp | 3 +- node/Revocation.hpp | 4 +-- node/Switch.hpp | 4 +-- node/Tag.hpp | 4 +-- node/World.hpp | 12 ++++---- 33 files changed, 155 insertions(+), 194 deletions(-) (limited to 'node') diff --git a/node/Array.hpp b/node/Array.hpp index 5c616475..ef2611e4 100644 --- a/node/Array.hpp +++ b/node/Array.hpp @@ -27,7 +27,6 @@ #ifndef ZT_ARRAY_HPP #define ZT_ARRAY_HPP -#include #include namespace ZeroTier { @@ -39,23 +38,23 @@ template class Array { public: - Array() throw() {} + Array() {} Array(const Array &a) { - for(std::size_t i=0;i reverse_iterator; typedef std::reverse_iterator const_reverse_iterator; - inline iterator begin() throw() { return data; } - inline iterator end() throw() { return &(data[S]); } - inline const_iterator begin() const throw() { return data; } - inline const_iterator end() const throw() { return &(data[S]); } + inline iterator begin() { return data; } + inline iterator end() { return &(data[S]); } + inline const_iterator begin() const { return data; } + inline const_iterator end() const { return &(data[S]); } - inline reverse_iterator rbegin() throw() { return reverse_iterator(begin()); } - inline reverse_iterator rend() throw() { return reverse_iterator(end()); } - inline const_reverse_iterator rbegin() const throw() { return const_reverse_iterator(begin()); } - inline const_reverse_iterator rend() const throw() { return const_reverse_iterator(end()); } + inline reverse_iterator rbegin() { return reverse_iterator(begin()); } + inline reverse_iterator rend() { return reverse_iterator(end()); } + inline const_reverse_iterator rbegin() const { return const_reverse_iterator(begin()); } + inline const_reverse_iterator rend() const { return const_reverse_iterator(end()); } + */ - inline std::size_t size() const throw() { return S; } - inline std::size_t max_size() const throw() { return S; } + inline unsigned long size() const { return S; } + inline unsigned long max_size() const { return S; } - inline reference operator[](const std::size_t n) throw() { return data[n]; } - inline const_reference operator[](const std::size_t n) const throw() { return data[n]; } + inline reference operator[](const std::size_t n) { return data[n]; } + inline const_reference operator[](const std::size_t n) const { return data[n]; } - inline reference front() throw() { return data[0]; } - inline const_reference front() const throw() { return data[0]; } - inline reference back() throw() { return data[S-1]; } - inline const_reference back() const throw() { return data[S-1]; } + inline reference front() { return data[0]; } + inline const_reference front() const { return data[0]; } + inline reference back() { return data[S-1]; } + inline const_reference back() const { return data[S-1]; } - inline bool operator==(const Array &k) const throw() + inline bool operator==(const Array &k) const { for(unsigned long i=0;i(const Array &k) const throw() { return (k < *this); } - inline bool operator<=(const Array &k) const throw() { return !(k < *this); } - inline bool operator>=(const Array &k) const throw() { return !(*this < k); } + inline bool operator!=(const Array &k) const { return !(*this == k); } + inline bool operator<(const Array &k) const { return std::lexicographical_compare(data,data + S,k.data,k.data + S); } + inline bool operator>(const Array &k) const { return (k < *this); } + inline bool operator<=(const Array &k) const { return !(k < *this); } + inline bool operator>=(const Array &k) const { return !(*this < k); } T data[S]; }; diff --git a/node/Buffer.hpp b/node/Buffer.hpp index 69ee1758..8979938f 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -95,7 +95,7 @@ public: Buffer(unsigned int l) { if (l > C) - throw std::out_of_range("Buffer: construct with size larger than capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; _l = l; } @@ -119,7 +119,7 @@ public: inline Buffer &operator=(const Buffer &b) { if (unlikely(b._l > C)) - throw std::out_of_range("Buffer: assignment from buffer larger than capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; if (C2 == C) { memcpy(this,&b,sizeof(Buffer)); } else { @@ -131,7 +131,7 @@ public: inline void copyFrom(const void *b,unsigned int l) { if (unlikely(l > C)) - throw std::out_of_range("Buffer: set from C array larger than capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; memcpy(_b,b,l); _l = l; } @@ -139,14 +139,14 @@ public: unsigned char operator[](const unsigned int i) const { if (unlikely(i >= _l)) - throw std::out_of_range("Buffer: [] beyond end of data"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; return (unsigned char)_b[i]; } unsigned char &operator[](const unsigned int i) { if (unlikely(i >= _l)) - throw std::out_of_range("Buffer: [] beyond end of data"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; return ((unsigned char *)_b)[i]; } @@ -166,13 +166,13 @@ public: unsigned char *field(unsigned int i,unsigned int l) { if (unlikely((i + l) > _l)) - throw std::out_of_range("Buffer: field() beyond end of data"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; return (unsigned char *)(_b + i); } const unsigned char *field(unsigned int i,unsigned int l) const { if (unlikely((i + l) > _l)) - throw std::out_of_range("Buffer: field() beyond end of data"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; return (const unsigned char *)(_b + i); } @@ -187,7 +187,7 @@ public: inline void setAt(unsigned int i,const T v) { if (unlikely((i + sizeof(T)) > _l)) - throw std::out_of_range("Buffer: setAt() beyond end of data"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; #ifdef ZT_NO_TYPE_PUNNING uint8_t *p = reinterpret_cast(_b + i); for(unsigned int x=1;x<=sizeof(T);++x) @@ -209,7 +209,7 @@ public: inline T at(unsigned int i) const { if (unlikely((i + sizeof(T)) > _l)) - throw std::out_of_range("Buffer: at() beyond end of data"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; #ifdef ZT_NO_TYPE_PUNNING T v = 0; const uint8_t *p = reinterpret_cast(_b + i); @@ -235,7 +235,7 @@ public: inline void append(const T v) { if (unlikely((_l + sizeof(T)) > C)) - throw std::out_of_range("Buffer: append beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; #ifdef ZT_NO_TYPE_PUNNING uint8_t *p = reinterpret_cast(_b + _l); for(unsigned int x=1;x<=sizeof(T);++x) @@ -257,7 +257,7 @@ public: inline void append(unsigned char c,unsigned int n) { if (unlikely((_l + n) > C)) - throw std::out_of_range("Buffer: append beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; for(unsigned int i=0;i C)) - throw std::out_of_range("Buffer: append beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; Utils::getSecureRandom(_b + _l,n); _l += n; } @@ -285,7 +285,7 @@ public: inline void append(const void *b,unsigned int l) { if (unlikely((_l + l) > C)) - throw std::out_of_range("Buffer: append beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; memcpy(_b + _l,b,l); _l += l; } @@ -311,7 +311,7 @@ public: { for(;;) { if (unlikely(_l >= C)) - throw std::out_of_range("Buffer: append beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; if (!(_b[_l++] = *(s++))) break; } @@ -343,7 +343,7 @@ public: inline char *appendField(unsigned int l) { if (unlikely((_l + l) > C)) - throw std::out_of_range("Buffer: append beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; char *r = _b + _l; _l += l; return r; @@ -360,7 +360,7 @@ public: inline void addSize(unsigned int i) { if (unlikely((i + _l) > C)) - throw std::out_of_range("Buffer: setSize to larger than capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; _l += i; } @@ -375,7 +375,7 @@ public: inline void setSize(const unsigned int i) { if (unlikely(i > C)) - throw std::out_of_range("Buffer: setSize to larger than capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; _l = i; } @@ -383,14 +383,14 @@ public: * Move everything after 'at' to the buffer's front and truncate * * @param at Truncate before this position - * @throw std::out_of_range Position is beyond size of buffer + * @throws std::out_of_range Position is beyond size of buffer */ inline void behead(const unsigned int at) { if (!at) return; if (unlikely(at > _l)) - throw std::out_of_range("Buffer: behead() beyond capacity"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; ::memmove(_b,_b + at,_l -= at); } @@ -399,13 +399,13 @@ public: * * @param start Starting position * @param length Length of block to erase - * @throw std::out_of_range Position plus length is beyond size of buffer + * @throws std::out_of_range Position plus length is beyond size of buffer */ inline void erase(const unsigned int at,const unsigned int length) { const unsigned int endr = at + length; if (unlikely(endr > _l)) - throw std::out_of_range("Buffer: erase() range beyond end of buffer"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; ::memmove(_b + at,_b + endr,_l - endr); _l -= length; } diff --git a/node/C25519.cpp b/node/C25519.cpp index 4158f1ba..2a5575aa 100644 --- a/node/C25519.cpp +++ b/node/C25519.cpp @@ -1998,7 +1998,6 @@ static inline void get_hram(unsigned char *hram, const unsigned char *sm, const ////////////////////////////////////////////////////////////////////////////// void C25519::agree(const C25519::Private &mine,const C25519::Public &their,void *keybuf,unsigned int keylen) - throw() { unsigned char rawkey[32]; unsigned char digest[64]; @@ -2015,7 +2014,6 @@ void C25519::agree(const C25519::Private &mine,const C25519::Public &their,void } void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPublic,const void *msg,unsigned int len,void *signature) - throw() { sc25519 sck, scs, scsk; ge25519 ger; @@ -2065,7 +2063,6 @@ void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPubli } bool C25519::verify(const C25519::Public &their,const void *msg,unsigned int len,const void *signature) - throw() { unsigned char t2[32]; ge25519 get1, get2; @@ -2096,7 +2093,6 @@ bool C25519::verify(const C25519::Public &their,const void *msg,unsigned int len } void C25519::_calcPubDH(C25519::Pair &kp) - throw() { // First 32 bytes of pub and priv are the keys for ECDH key // agreement. This generates the public portion from the private. @@ -2104,7 +2100,6 @@ void C25519::_calcPubDH(C25519::Pair &kp) } void C25519::_calcPubED(C25519::Pair &kp) - throw() { unsigned char extsk[64]; sc25519 scsk; diff --git a/node/C25519.hpp b/node/C25519.hpp index da9ba665..950c7fed 100644 --- a/node/C25519.hpp +++ b/node/C25519.hpp @@ -69,7 +69,6 @@ public: * Generate a C25519 elliptic curve key pair */ static inline Pair generate() - throw() { Pair kp; Utils::getSecureRandom(kp.priv.data,(unsigned int)kp.priv.size()); @@ -93,7 +92,6 @@ public: */ template static inline Pair generateSatisfying(F cond) - throw() { Pair kp; void *const priv = (void *)kp.priv.data; @@ -118,13 +116,8 @@ public: * @param keybuf Buffer to fill * @param keylen Number of key bytes to generate */ - static void agree(const Private &mine,const Public &their,void *keybuf,unsigned int keylen) - throw(); - static inline void agree(const Pair &mine,const Public &their,void *keybuf,unsigned int keylen) - throw() - { - agree(mine.priv,their,keybuf,keylen); - } + static void agree(const Private &mine,const Public &their,void *keybuf,unsigned int keylen); + static inline void agree(const Pair &mine,const Public &their,void *keybuf,unsigned int keylen) { agree(mine.priv,their,keybuf,keylen); } /** * Sign a message with a sender's key pair @@ -145,13 +138,8 @@ public: * @param len Length of message in bytes * @param signature Buffer to fill with signature -- MUST be 96 bytes in length */ - static void sign(const Private &myPrivate,const Public &myPublic,const void *msg,unsigned int len,void *signature) - throw(); - static inline void sign(const Pair &mine,const void *msg,unsigned int len,void *signature) - throw() - { - sign(mine.priv,mine.pub,msg,len,signature); - } + static void sign(const Private &myPrivate,const Public &myPublic,const void *msg,unsigned int len,void *signature); + static inline void sign(const Pair &mine,const void *msg,unsigned int len,void *signature) { sign(mine.priv,mine.pub,msg,len,signature); } /** * Sign a message with a sender's key pair @@ -163,14 +151,12 @@ public: * @return Signature */ static inline Signature sign(const Private &myPrivate,const Public &myPublic,const void *msg,unsigned int len) - throw() { Signature sig; sign(myPrivate,myPublic,msg,len,sig.data); return sig; } static inline Signature sign(const Pair &mine,const void *msg,unsigned int len) - throw() { Signature sig; sign(mine.priv,mine.pub,msg,len,sig.data); @@ -186,8 +172,7 @@ public: * @param signature 96-byte signature * @return True if signature is valid and the message is authentic and unmodified */ - static bool verify(const Public &their,const void *msg,unsigned int len,const void *signature) - throw(); + static bool verify(const Public &their,const void *msg,unsigned int len,const void *signature); /** * Verify a message's signature @@ -199,7 +184,6 @@ public: * @return True if signature is valid and the message is authentic and unmodified */ static inline bool verify(const Public &their,const void *msg,unsigned int len,const Signature &signature) - throw() { return verify(their,msg,len,signature.data); } @@ -207,13 +191,11 @@ public: private: // derive first 32 bytes of kp.pub from first 32 bytes of kp.priv // this is the ECDH key - static void _calcPubDH(Pair &kp) - throw(); + static void _calcPubDH(Pair &kp); // derive 2nd 32 bytes of kp.pub from 2nd 32 bytes of kp.priv // this is the Ed25519 sign/verify key - static void _calcPubED(Pair &kp) - throw(); + static void _calcPubED(Pair &kp); }; } // namespace ZeroTier diff --git a/node/Capability.hpp b/node/Capability.hpp index 8d4b9085..d64625e6 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -420,24 +420,24 @@ public: const unsigned int rc = b.template at(p); p += 2; if (rc > ZT_MAX_CAPABILITY_RULES) - throw std::runtime_error("rule overflow"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; deserializeRules(b,p,_rules,_ruleCount,rc); _maxCustodyChainLength = (unsigned int)b[p++]; if ((_maxCustodyChainLength < 1)||(_maxCustodyChainLength > ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) - throw std::runtime_error("invalid max custody chain length"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; for(unsigned int i=0;;++i) { const Address to(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (!to) break; if ((i >= _maxCustodyChainLength)||(i >= ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH)) - throw std::runtime_error("unterminated custody chain"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; _custody[i].to = to; _custody[i].from.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (b[p++] == 1) { if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) - throw std::runtime_error("invalid signature"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; p += 2; memcpy(_custody[i].signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else { @@ -447,7 +447,7 @@ public: p += 2 + b.template at(p); if (p > b.size()) - throw std::runtime_error("extended field overflow"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; return (p - startAt); } diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 739d5390..3ffa814f 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -305,14 +305,14 @@ public: _signedBy.zero(); if (b[p++] != 1) - throw std::invalid_argument("invalid object"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE; unsigned int numq = b.template at(p); p += sizeof(uint16_t); uint64_t lastId = 0; for(unsigned int i=0;i(p); if (qid < lastId) - throw std::invalid_argument("qualifiers not sorted"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_BAD_ENCODING; else lastId = qid; if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) { _qualifiers[_qualifierCount].id = qid; @@ -321,7 +321,7 @@ public: p += 24; ++_qualifierCount; } else { - throw std::invalid_argument("too many qualifiers"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; } } @@ -359,7 +359,7 @@ private: uint64_t id; uint64_t value; uint64_t maxDelta; - inline bool operator<(const _Qualifier &q) const throw() { return (id < q.id); } // sort order + inline bool operator<(const _Qualifier &q) const { return (id < q.id); } // sort order }; Address _signedBy; diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 95039a2d..775324e6 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -207,7 +207,7 @@ public: _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (b[p++] == 1) { if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) - throw std::runtime_error("invalid signature length"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; p += 2; memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else { @@ -216,7 +216,7 @@ public: p += 2 + b.template at(p); if (p > b.size()) - throw std::runtime_error("extended field overflow"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; return (p - startAt); } diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp index 92a71bc0..3007f1dc 100644 --- a/node/CertificateOfRepresentation.hpp +++ b/node/CertificateOfRepresentation.hpp @@ -164,14 +164,14 @@ public: p += 2; memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; - } else throw std::runtime_error("invalid signature"); + } else throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; } else { p += 2 + b.template at(p); } p += 2 + b.template at(p); if (p > b.size()) - throw std::runtime_error("extended field overflow"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; return (p - startAt); } diff --git a/node/Constants.hpp b/node/Constants.hpp index 12bf8d28..3f050ead 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -478,4 +478,13 @@ #define ZT_ETHERTYPE_IPX_B 0x8138 #define ZT_ETHERTYPE_IPV6 0x86dd +#define ZT_EXCEPTION_OUT_OF_BOUNDS 100 +#define ZT_EXCEPTION_OUT_OF_MEMORY 101 +#define ZT_EXCEPTION_PRIVATE_KEY_REQUIRED 102 +#define ZT_EXCEPTION_INVALID_ARGUMENT 103 +#define ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE 200 +#define ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW 201 +#define ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN 202 +#define ZT_EXCEPTION_INVALID_SERIALIZED_DATA_BAD_ENCODING 203 + #endif diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index b702f608..eff5b62a 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -119,7 +119,7 @@ public: _s(0) { if (!_t) - throw std::bad_alloc(); + throw ZT_EXCEPTION_OUT_OF_MEMORY; for(unsigned long i=0;i diff --git a/node/Identity.cpp b/node/Identity.cpp index a972d60d..72bea75d 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -83,10 +83,9 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub // threshold value. struct _Identity_generate_cond { - _Identity_generate_cond() throw() {} - _Identity_generate_cond(unsigned char *sb,char *gm) throw() : digest(sb),genmem(gm) {} + _Identity_generate_cond() {} + _Identity_generate_cond(unsigned char *sb,char *gm) : digest(sb),genmem(gm) {} inline bool operator()(const C25519::Pair &kp) const - throw() { _computeMemoryHardHash(kp.pub.data,(unsigned int)kp.pub.size(),digest,genmem); return (digest[0] < ZT_IDENTITY_GEN_HASHCASH_FIRST_BYTE_LESS_THAN); diff --git a/node/Identity.hpp b/node/Identity.hpp index 5804b9f8..3d4d9385 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -71,7 +71,7 @@ public: _privateKey((C25519::Private *)0) { if (!fromString(str)) - throw std::invalid_argument(std::string("invalid string-serialized identity: ") + str); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE; } template @@ -121,7 +121,7 @@ public: /** * @return True if this identity contains a private key */ - inline bool hasPrivate() const throw() { return (_privateKey != (C25519::Private *)0); } + inline bool hasPrivate() const { return (_privateKey != (C25519::Private *)0); } /** * Compute the SHA512 hash of our private key (if we have one) @@ -145,11 +145,10 @@ public: * @param len Length of data */ inline C25519::Signature sign(const void *data,unsigned int len) const - throw(std::runtime_error) { if (_privateKey) return C25519::sign(*_privateKey,_publicKey,data,len); - throw std::runtime_error("sign() requires a private key"); + throw ZT_EXCEPTION_PRIVATE_KEY_REQUIRED; } /** @@ -203,7 +202,7 @@ public: /** * @return This identity's address */ - inline const Address &address() const throw() { return _address; } + inline const Address &address() const { return _address; } /** * Serialize this identity (binary) @@ -248,7 +247,7 @@ public: p += ZT_ADDRESS_LENGTH; if (b[p++] != 0) - throw std::invalid_argument("unsupported identity type"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE; memcpy(_publicKey.data,b.field(p,(unsigned int)_publicKey.size()),(unsigned int)_publicKey.size()); p += (unsigned int)_publicKey.size(); @@ -256,7 +255,7 @@ public: unsigned int privateKeyLength = (unsigned int)b[p++]; if (privateKeyLength) { if (privateKeyLength != ZT_C25519_PRIVATE_KEY_LEN) - throw std::invalid_argument("invalid private key"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; _privateKey = new C25519::Private(); memcpy(_privateKey->data,b.field(p,ZT_C25519_PRIVATE_KEY_LEN),ZT_C25519_PRIVATE_KEY_LEN); p += ZT_C25519_PRIVATE_KEY_LEN; @@ -306,14 +305,14 @@ public: /** * @return True if this identity contains something */ - inline operator bool() const throw() { return (_address); } - - inline bool operator==(const Identity &id) const throw() { return ((_address == id._address)&&(_publicKey == id._publicKey)); } - inline bool operator<(const Identity &id) const throw() { return ((_address < id._address)||((_address == id._address)&&(_publicKey < id._publicKey))); } - inline bool operator!=(const Identity &id) const throw() { return !(*this == id); } - inline bool operator>(const Identity &id) const throw() { return (id < *this); } - inline bool operator<=(const Identity &id) const throw() { return !(id < *this); } - inline bool operator>=(const Identity &id) const throw() { return !(*this < id); } + inline operator bool() const { return (_address); } + + inline bool operator==(const Identity &id) const { return ((_address == id._address)&&(_publicKey == id._publicKey)); } + inline bool operator<(const Identity &id) const { return ((_address < id._address)||((_address == id._address)&&(_publicKey < id._publicKey))); } + inline bool operator!=(const Identity &id) const { return !(*this == id); } + inline bool operator>(const Identity &id) const { return (id < *this); } + inline bool operator<=(const Identity &id) const { return !(id < *this); } + inline bool operator>=(const Identity &id) const { return !(*this < id); } private: Address _address; diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 692c63df..45a0166d 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -118,7 +118,7 @@ public: /** * @return Time of packet receipt / start of decode */ - inline uint64_t receiveTime() const throw() { return _receiveTime; } + inline uint64_t receiveTime() const { return _receiveTime; } private: // These are called internally to handle packet contents once it has diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index dd055084..61cdb05e 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -521,7 +521,7 @@ struct InetAddress : public sockaddr_storage reinterpret_cast(this)->sin_port = Utils::hton(b.template at(p)); p += 2; break; default: - throw std::invalid_argument("invalid serialized InetAddress"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_BAD_ENCODING; } return (p - startAt); } diff --git a/node/MAC.hpp b/node/MAC.hpp index 52388d59..a179fd4f 100644 --- a/node/MAC.hpp +++ b/node/MAC.hpp @@ -191,7 +191,7 @@ public: * @param i Value from 0 to 5 (inclusive) * @return Byte at said position (address interpreted in big-endian order) */ - inline unsigned char operator[](unsigned int i) const throw() { return (unsigned char)((_m >> (40 - (i * 8))) & 0xff); } + inline unsigned char operator[](unsigned int i) const { return (unsigned char)((_m >> (40 - (i * 8))) & 0xff); } /** * @return 6, which is the number of bytes in a MAC, for container compliance diff --git a/node/MulticastGroup.hpp b/node/MulticastGroup.hpp index f56c675b..6039d3c4 100644 --- a/node/MulticastGroup.hpp +++ b/node/MulticastGroup.hpp @@ -52,15 +52,13 @@ namespace ZeroTier { class MulticastGroup { public: - MulticastGroup() - throw() : + MulticastGroup() : _mac(), _adi(0) { } - MulticastGroup(const MAC &m,uint32_t a) - throw() : + MulticastGroup(const MAC &m,uint32_t a) : _mac(m), _adi(a) { @@ -73,7 +71,6 @@ public: * @return Multicat group for ARP/NDP */ static inline MulticastGroup deriveMulticastGroupForAddressResolution(const InetAddress &ip) - throw() { if (ip.isV4()) { // IPv4 wants broadcast MACs, so we shove the V4 address itself into @@ -95,18 +92,18 @@ public: /** * @return Multicast address */ - inline const MAC &mac() const throw() { return _mac; } + inline const MAC &mac() const { return _mac; } /** * @return Additional distinguishing information */ - inline uint32_t adi() const throw() { return _adi; } + inline uint32_t adi() const { return _adi; } - inline unsigned long hashCode() const throw() { return (_mac.hashCode() ^ (unsigned long)_adi); } + inline unsigned long hashCode() const { return (_mac.hashCode() ^ (unsigned long)_adi); } - inline bool operator==(const MulticastGroup &g) const throw() { return ((_mac == g._mac)&&(_adi == g._adi)); } - inline bool operator!=(const MulticastGroup &g) const throw() { return ((_mac != g._mac)||(_adi != g._adi)); } - inline bool operator<(const MulticastGroup &g) const throw() + inline bool operator==(const MulticastGroup &g) const { return ((_mac == g._mac)&&(_adi == g._adi)); } + inline bool operator!=(const MulticastGroup &g) const { return ((_mac != g._mac)||(_adi != g._adi)); } + inline bool operator<(const MulticastGroup &g) const { if (_mac < g._mac) return true; @@ -114,9 +111,9 @@ public: return (_adi < g._adi); return false; } - inline bool operator>(const MulticastGroup &g) const throw() { return (g < *this); } - inline bool operator<=(const MulticastGroup &g) const throw() { return !(g < *this); } - inline bool operator>=(const MulticastGroup &g) const throw() { return !(*this < g); } + inline bool operator>(const MulticastGroup &g) const { return (g < *this); } + inline bool operator<=(const MulticastGroup &g) const { return !(g < *this); } + inline bool operator>=(const MulticastGroup &g) const { return !(*this < g); } private: MAC _mac; diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 2186e9c3..69a6645d 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -64,8 +64,8 @@ private: uint64_t nwid; MulticastGroup mg; - inline bool operator==(const Key &k) const throw() { return ((nwid == k.nwid)&&(mg == k.mg)); } - inline unsigned long hashCode() const throw() { return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32))); } + inline bool operator==(const Key &k) const { return ((nwid == k.nwid)&&(mg == k.mg)); } + inline unsigned long hashCode() const { return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32))); } }; struct MulticastGroupMember diff --git a/node/Mutex.hpp b/node/Mutex.hpp index 6f1d3471..854f321a 100644 --- a/node/Mutex.hpp +++ b/node/Mutex.hpp @@ -41,7 +41,6 @@ class Mutex : NonCopyable { public: Mutex() - throw() { pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0); } @@ -52,25 +51,21 @@ public: } inline void lock() - throw() { pthread_mutex_lock(&_mh); } inline void unlock() - throw() { pthread_mutex_unlock(&_mh); } inline void lock() const - throw() { (const_cast (this))->lock(); } inline void unlock() const - throw() { (const_cast (this))->unlock(); } @@ -81,15 +76,13 @@ public: class Lock : NonCopyable { public: - Lock(Mutex &m) - throw() : + Lock(Mutex &m) : _m(&m) { m.lock(); } - Lock(const Mutex &m) - throw() : + Lock(const Mutex &m) : _m(const_cast(&m)) { _m->lock(); @@ -123,7 +116,6 @@ class Mutex : NonCopyable { public: Mutex() - throw() { InitializeCriticalSection(&_cs); } @@ -134,25 +126,21 @@ public: } inline void lock() - throw() { EnterCriticalSection(&_cs); } inline void unlock() - throw() { LeaveCriticalSection(&_cs); } inline void lock() const - throw() { (const_cast (this))->lock(); } inline void unlock() const - throw() { (const_cast (this))->unlock(); } @@ -160,15 +148,13 @@ public: class Lock : NonCopyable { public: - Lock(Mutex &m) - throw() : + Lock(Mutex &m) : _m(&m) { m.lock(); } - Lock(const Mutex &m) - throw() : + Lock(const Mutex &m) : _m(const_cast(&m)) { _m->lock(); diff --git a/node/Network.hpp b/node/Network.hpp index be5f1a12..d4d217f2 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -76,7 +76,7 @@ public: /** * Compute primary controller device ID from network ID */ - static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); } + static inline Address controllerFor(uint64_t nwid) { return Address(nwid >> 24); } /** * Construct a new network @@ -98,7 +98,7 @@ public: inline Address controller() const { return Address(_id >> 24); } inline bool multicastEnabled() const { return (_config.multicastLimit > 0); } inline bool hasConfig() const { return (_config); } - inline uint64_t lastConfigUpdate() const throw() { return _lastConfigUpdate; } + inline uint64_t lastConfigUpdate() const { return _lastConfigUpdate; } inline ZT_VirtualNetworkStatus status() const { Mutex::Lock _l(_lock); return _status(); } inline const NetworkConfig &config() const { return _config; } inline const MAC &mac() const { return _mac; } diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 8b3b3619..fb48edc9 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -262,32 +262,32 @@ public: /** * @return True if passive bridging is allowed (experimental) */ - inline bool allowPassiveBridging() const throw() { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING) != 0); } + inline bool allowPassiveBridging() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING) != 0); } /** * @return True if broadcast (ff:ff:ff:ff:ff:ff) address should work on this network */ - inline bool enableBroadcast() const throw() { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST) != 0); } + inline bool enableBroadcast() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST) != 0); } /** * @return True if IPv6 NDP emulation should be allowed for certain "magic" IPv6 address patterns */ - inline bool ndpEmulation() const throw() { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0); } + inline bool ndpEmulation() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0); } /** * @return True if frames should not be compressed */ - inline bool disableCompression() const throw() { return ((this->flags & ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION) != 0); } + inline bool disableCompression() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION) != 0); } /** * @return Network type is public (no access control) */ - inline bool isPublic() const throw() { return (this->type == ZT_NETWORK_TYPE_PUBLIC); } + inline bool isPublic() const { return (this->type == ZT_NETWORK_TYPE_PUBLIC); } /** * @return Network type is private (certificate access control) */ - inline bool isPrivate() const throw() { return (this->type == ZT_NETWORK_TYPE_PRIVATE); } + inline bool isPrivate() const { return (this->type == ZT_NETWORK_TYPE_PRIVATE); } /** * @return ZeroTier addresses of devices on this network designated as active bridges @@ -361,7 +361,7 @@ public: /** * @return True if this network config is non-NULL */ - inline operator bool() const throw() { return (networkId != 0); } + inline operator bool() const { return (networkId != 0); } inline bool operator==(const NetworkConfig &nc) const { return (memcmp(this,&nc,sizeof(NetworkConfig)) == 0); } inline bool operator!=(const NetworkConfig &nc) const { return (!(*this == nc)); } diff --git a/node/Node.cpp b/node/Node.cpp index dea0f5ca..f3339068 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -64,7 +64,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 _lastHousekeepingRun(0) { if (callbacks->version != 0) - throw std::runtime_error("callbacks struct version mismatch"); + throw ZT_EXCEPTION_INVALID_ARGUMENT; memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); // Initialize non-cryptographic PRNG from a good random source diff --git a/node/Node.hpp b/node/Node.hpp index e60da1ad..9658174f 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -114,7 +114,7 @@ public: // Internal functions ------------------------------------------------------ - inline uint64_t now() const throw() { return _now; } + inline uint64_t now() const { return _now; } inline bool putPacket(void *tPtr,const int64_t localSocket,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { @@ -182,7 +182,7 @@ public: inline int configureVirtualNetworkPort(void *tPtr,uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast(this),_uPtr,tPtr,nwid,nuptr,op,nc); } - inline bool online() const throw() { return _online; } + inline bool online() const { return _online; } inline int stateObjectGet(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],void *const data,const unsigned int maxlen) { return _cb.stateGetFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,maxlen); } inline void stateObjectPut(void *const tPtr,ZT_StateObjectType type,const uint64_t id[2],const void *const data,const unsigned int len) { _cb.statePutFunction(reinterpret_cast(this),_uPtr,tPtr,type,id,data,(int)len); } diff --git a/node/NonCopyable.hpp b/node/NonCopyable.hpp index 25c71b1c..5e8c753d 100644 --- a/node/NonCopyable.hpp +++ b/node/NonCopyable.hpp @@ -35,7 +35,7 @@ namespace ZeroTier { class NonCopyable { protected: - NonCopyable() throw() {} + NonCopyable() {} private: NonCopyable(const NonCopyable&); const NonCopyable& operator=(const NonCopyable&); diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 0c988804..486b66ff 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -90,18 +90,18 @@ public: /** * @return Multicast creation time */ - inline uint64_t timestamp() const throw() { return _timestamp; } + inline uint64_t timestamp() const { return _timestamp; } /** * @param now Current time * @return True if this multicast is expired (has exceeded transmit timeout) */ - inline bool expired(uint64_t now) const throw() { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); } + inline bool expired(uint64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); } /** * @return True if this outbound multicast has been sent to enough peers */ - inline bool atLimit() const throw() { return (_alreadySentTo.size() >= _limit); } + inline bool atLimit() const { return (_alreadySentTo.size() >= _limit); } /** * Just send without checking log diff --git a/node/Packet.hpp b/node/Packet.hpp index 4192b4e3..97c9774e 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -424,8 +424,7 @@ public: } template - Fragment(const Buffer &b) - throw(std::out_of_range) : + Fragment(const Buffer &b) : Buffer(b) { } @@ -443,10 +442,8 @@ public: * @param fragLen Length of fragment in bytes * @param fragNo Which fragment (>= 1, since 0 is Packet with end chopped off) * @param fragTotal Total number of fragments (including 0) - * @throws std::out_of_range Packet size would exceed buffer */ Fragment(const Packet &p,unsigned int fragStart,unsigned int fragLen,unsigned int fragNo,unsigned int fragTotal) - throw(std::out_of_range) { init(p,fragStart,fragLen,fragNo,fragTotal); } @@ -459,13 +456,11 @@ public: * @param fragLen Length of fragment in bytes * @param fragNo Which fragment (>= 1, since 0 is Packet with end chopped off) * @param fragTotal Total number of fragments (including 0) - * @throws std::out_of_range Packet size would exceed buffer */ inline void init(const Packet &p,unsigned int fragStart,unsigned int fragLen,unsigned int fragNo,unsigned int fragTotal) - throw(std::out_of_range) { if ((fragStart + fragLen) > p.size()) - throw std::out_of_range("Packet::Fragment: tried to construct fragment of packet past its length"); + throw ZT_EXCEPTION_OUT_OF_BOUNDS; setSize(fragLen + ZT_PROTO_MIN_FRAGMENT_LENGTH); // NOTE: this copies both the IV/packet ID and the destination address. diff --git a/node/Peer.cpp b/node/Peer.cpp index d362be9f..986e52ef 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -61,7 +61,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _credentialsCutoffCount(0) { if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) - throw std::runtime_error("new peer identity key agreement failed"); + throw ZT_EXCEPTION_INVALID_ARGUMENT; } void Peer::received( diff --git a/node/Peer.hpp b/node/Peer.hpp index 6d00e3e6..c6423a59 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -81,12 +81,12 @@ public: /** * @return This peer's ZT address (short for identity().address()) */ - inline const Address &address() const throw() { return _id.address(); } + inline const Address &address() const { return _id.address(); } /** * @return This peer's identity */ - inline const Identity &identity() const throw() { return _id; } + inline const Identity &identity() const { return _id; } /** * Log receipt of an authenticated packet diff --git a/node/Poly1305.cpp b/node/Poly1305.cpp index 46a51390..eceb57b3 100644 --- a/node/Poly1305.cpp +++ b/node/Poly1305.cpp @@ -121,7 +121,6 @@ static inline int crypto_onetimeauth(unsigned char *out,const unsigned char *in, } void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key) - throw() { crypto_onetimeauth((unsigned char *)auth,(const unsigned char *)data,len,(const unsigned char *)key); } @@ -623,7 +622,6 @@ poly1305_update(poly1305_context *ctx, const unsigned char *m, size_t bytes) { } // anonymous namespace void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key) - throw() { poly1305_context ctx; poly1305_init(&ctx,reinterpret_cast(key)); diff --git a/node/Poly1305.hpp b/node/Poly1305.hpp index ff709983..0bdfa74f 100644 --- a/node/Poly1305.hpp +++ b/node/Poly1305.hpp @@ -54,8 +54,7 @@ public: * @param len Length of data to authenticate in bytes * @param key 32-byte one-time use key to authenticate data (must not be reused) */ - static void compute(void *auth,const void *data,unsigned int len,const void *key) - throw(); + static void compute(void *auth,const void *data,unsigned int len,const void *key); }; } // namespace ZeroTier diff --git a/node/Revocation.hpp b/node/Revocation.hpp index e8f5d00d..a28da0ab 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -168,14 +168,14 @@ public: p += 2; memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; - } else throw std::runtime_error("invalid signature"); + } else throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; } else { p += 2 + b.template at(p); } p += 2 + b.template at(p); if (p > b.size()) - throw std::runtime_error("extended field overflow"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; return (p - startAt); } diff --git a/node/Switch.hpp b/node/Switch.hpp index cebe9e67..346aaca3 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -222,8 +222,8 @@ private: y = a2.toInt(); } } - inline unsigned long hashCode() const throw() { return ((unsigned long)x ^ (unsigned long)y); } - inline bool operator==(const _LastUniteKey &k) const throw() { return ((x == k.x)&&(y == k.y)); } + inline unsigned long hashCode() const { return ((unsigned long)x ^ (unsigned long)y); } + inline bool operator==(const _LastUniteKey &k) const { return ((x == k.x)&&(y == k.y)); } uint64_t x,y; }; Hashtable< _LastUniteKey,uint64_t > _lastUniteAttempt; // key is always sorted in ascending order, for set-like behavior diff --git a/node/Tag.hpp b/node/Tag.hpp index 746ade26..5fbfb278 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -161,7 +161,7 @@ public: _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; if (b[p++] == 1) { if (b.template at(p) != ZT_C25519_SIGNATURE_LEN) - throw std::runtime_error("invalid signature length"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; p += 2; memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; } else { @@ -170,7 +170,7 @@ public: p += 2 + b.template at(p); if (p > b.size()) - throw std::runtime_error("extended field overflow"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; return (p - startAt); } diff --git a/node/World.hpp b/node/World.hpp index 003d70e3..71ab6e7c 100644 --- a/node/World.hpp +++ b/node/World.hpp @@ -110,9 +110,9 @@ public: Identity identity; std::vector stableEndpoints; - inline bool operator==(const Root &r) const throw() { return ((identity == r.identity)&&(stableEndpoints == r.stableEndpoints)); } - inline bool operator!=(const Root &r) const throw() { return (!(*this == r)); } - inline bool operator<(const Root &r) const throw() { return (identity < r.identity); } // for sorting + inline bool operator==(const Root &r) const { return ((identity == r.identity)&&(stableEndpoints == r.stableEndpoints)); } + inline bool operator!=(const Root &r) const { return (!(*this == r)); } + inline bool operator<(const Root &r) const { return (identity < r.identity); } // for sorting }; /** @@ -212,7 +212,7 @@ public: case TYPE_PLANET: _type = TYPE_PLANET; break; case TYPE_MOON: _type = TYPE_MOON; break; default: - throw std::invalid_argument("invalid world type"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE; } _id = b.template at(p); p += 8; @@ -221,14 +221,14 @@ public: memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN; const unsigned int numRoots = (unsigned int)b[p++]; if (numRoots > ZT_WORLD_MAX_ROOTS) - throw std::invalid_argument("too many roots in World"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; for(unsigned int k=0;k ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT) - throw std::invalid_argument("too many stable endpoints in World/Root"); + throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; for(unsigned int kk=0;kk Date: Mon, 17 Jul 2017 14:24:57 -0700 Subject: Cleanup. --- node/Buffer.hpp | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'node') diff --git a/node/Buffer.hpp b/node/Buffer.hpp index 8979938f..7b91e72f 100644 --- a/node/Buffer.hpp +++ b/node/Buffer.hpp @@ -110,11 +110,6 @@ public: copyFrom(b,l); } - Buffer(const std::string &s) - { - copyFrom(s.data(),s.length()); - } - template inline Buffer &operator=(const Buffer &b) { @@ -290,17 +285,6 @@ public: _l += l; } - /** - * Append a string - * - * @param s String to append - * @throws std::out_of_range Attempt to append beyond capacity - */ - inline void append(const std::string &s) - { - append(s.data(),(unsigned int)s.length()); - } - /** * Append a C string including null termination byte * -- cgit v1.2.3 From 1685659e37f568c727580634e412674cc266ff31 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Jul 2017 17:02:50 -0700 Subject: Remote tracing works. --- controller/EmbeddedNetworkController.cpp | 12 ++++++++--- node/IncomingPacket.cpp | 4 +++- node/NetworkConfig.hpp | 29 -------------------------- node/Packet.hpp | 4 ---- node/Trace.cpp | 35 ++++++++++++++++---------------- node/Trace.hpp | 2 +- 6 files changed, 30 insertions(+), 56 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c2024962..07ab5168 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1089,7 +1089,9 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) { try { std::vector nw4m(_db.networksForMember(rt.origin)); - if (nw4m.empty()) // ignore these for unknown members + + // Ignore remote traces from members we don't know about + if (nw4m.empty()) return; // Convert Dictionary into JSON object @@ -1133,7 +1135,8 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } - bool accept = false; + bool accept = true; + /* for(std::vector::const_iterator nwid(nw4m.begin());nwid!=nw4m.end();++nwid) { json nconf; if (_db.getNetwork(*nwid,nconf)) { @@ -1153,9 +1156,10 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } catch ( ... ) {} // ignore missing fields or other errors, drop trace message } } + */ if (accept) { char p[128]; - OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx_%.16llx.json",rt.origin,OSUtils::now()); + OSUtils::ztsnprintf(p,sizeof(p),"trace/%.10llx-%.10llx-%.16llx",_signingId.address().toInt(),rt.origin,OSUtils::now()); _db.writeRaw(p,OSUtils::jsonDump(d)); } } catch ( ... ) { @@ -1419,6 +1423,8 @@ void EmbeddedNetworkController::_request( rtt = OSUtils::jsonString(network["remoteTraceTarget"],""); if (rtt.length() == 10) { nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str())); + } else { + nc->remoteTraceTarget = _signingId.address(); } } diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index be3d082b..51955bf3 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -115,6 +115,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) case Packet::VERB_MULTICAST_FRAME: return _doMULTICAST_FRAME(RR,tPtr,peer); case Packet::VERB_PUSH_DIRECT_PATHS: return _doPUSH_DIRECT_PATHS(RR,tPtr,peer); case Packet::VERB_USER_MESSAGE: return _doUSER_MESSAGE(RR,tPtr,peer); + case Packet::VERB_REMOTE_TRACE: return _doREMOTE_TRACE(RR,tPtr,peer); } } else { RR->sw->requestWhois(tPtr,sourceAddress); @@ -1172,8 +1173,9 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con while (ptr < eof) { if (!*ptr) { // end of string rt.len = (unsigned int)(ptr - rt.data); - if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) + if ((rt.len > 0)&&(rt.len <= ZT_MAX_REMOTE_TRACE_SIZE)) { RR->node->postEvent(tPtr,ZT_EVENT_REMOTE_TRACE,&rt); + } rt.data = const_cast(++ptr); // start of next string, if any } else { ++ptr; diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index fb48edc9..3fd5ddac 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -410,35 +410,6 @@ public: return (Tag *)0; } - /* - inline void dump() const - { - printf("networkId==%.16llx\n",networkId); - printf("timestamp==%llu\n",timestamp); - printf("credentialTimeMaxDelta==%llu\n",credentialTimeMaxDelta); - printf("revision==%llu\n",revision); - printf("issuedTo==%.10llx\n",issuedTo.toInt()); - printf("multicastLimit==%u\n",multicastLimit); - printf("flags=%.8lx\n",(unsigned long)flags); - printf("specialistCount==%u\n",specialistCount); - for(unsigned int i=0;i(&(routes[i].target))->toString().c_str()); - printf(" routes[i].via==%s\n",reinterpret_cast(&(routes[i].via))->toIpString().c_str()); - printf(" routes[i].flags==%.4x\n",(unsigned int)routes[i].flags); - printf(" routes[i].metric==%u\n",(unsigned int)routes[i].metric); - } - printf("staticIpCount==%u\n",staticIpCount); - for(unsigned int i=0;ilocalSocket()); } d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id()); - _send(tPtr,d,network); + _send(tPtr,d,*network); } void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &network,const SharedPtr &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac,const char *reason) @@ -161,7 +161,7 @@ void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) @@ -218,7 +218,7 @@ void Trace::networkConfigRequestSent(void *const tPtr,const Network &network,con d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT_S); d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id()); d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID,controller); - _send(tPtr,d,0); + _send(tPtr,d,network); } void Trace::networkFilter( @@ -259,7 +259,7 @@ void Trace::networkFilter( d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen); if (frameLen > 0) d.add(ZT_REMOTE_TRACE_FIELD__FRAME_DATA,(const char *)frameData,(frameLen > 256) ? (int)256 : (int)frameLen); - _send(tPtr,d,network.id()); + _send(tPtr,d,network); } void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason) @@ -273,7 +273,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason) @@ -287,7 +287,7 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c, d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason) @@ -313,7 +313,7 @@ void Trace::credentialRejected(void *const tPtr,const Capability &c,const char * d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) @@ -328,7 +328,7 @@ void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *reason) @@ -341,7 +341,7 @@ void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char * d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); if (reason) d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c) @@ -353,7 +353,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) @@ -365,7 +365,7 @@ void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c) @@ -387,7 +387,7 @@ void Trace::credentialAccepted(void *const tPtr,const Capability &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const Tag &c) @@ -400,7 +400,7 @@ void Trace::credentialAccepted(void *const tPtr,const Tag &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::credentialAccepted(void *const tPtr,const Revocation &c) @@ -411,7 +411,7 @@ void Trace::credentialAccepted(void *const tPtr,const Revocation &c) d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); - _send(tPtr,d,0); + _send(tPtr,d,c.networkId()); } void Trace::_send(void *const tPtr,const Dictionary &d) @@ -434,7 +434,6 @@ void Trace::_send(void *const tPtr,const Dictionary &d } } _traceMsgBuf[i] = (char)0; - //printf("%s\n",_traceMsgBuf); RR->node->postEvent(tPtr,ZT_EVENT_TRACE,_traceMsgBuf); #endif @@ -461,11 +460,11 @@ void Trace::_send(void *const tPtr,const Dictionary &d } } -void Trace::_send(void *const tPtr,const Dictionary &d,const SharedPtr &network) +void Trace::_send(void *const tPtr,const Dictionary &d,const Network &network) { _send(tPtr,d); - if ((network)&&(network->config().remoteTraceTarget)) { - Packet outp(network->config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE); + if (network.config().remoteTraceTarget) { + Packet outp(network.config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE); outp.appendCString(d.data()); outp.compress(); RR->sw->send(tPtr,outp,true); diff --git a/node/Trace.hpp b/node/Trace.hpp index 7fe48cdd..d66d0871 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -154,7 +154,7 @@ private: void _send(void *const tPtr,const Dictionary &d); void _send(void *const tPtr,const Dictionary &d,const uint64_t networkId); - void _send(void *const tPtr,const Dictionary &d,const SharedPtr &network); + void _send(void *const tPtr,const Dictionary &d,const Network &network); #ifdef ZT_TRACE char _traceMsgBuf[4096]; -- cgit v1.2.3 From 727ccb112543f3c44da3d094fa755e3a5d25cc3e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Jul 2017 13:57:37 -0700 Subject: Cleanup and stdin/stdout harness mode for controller. --- controller/JSONDB.cpp | 185 +++++++++++++++++++++++++++++++------------- controller/JSONDB.hpp | 3 + node/IncomingPacket.cpp | 2 - service/SoftwareUpdater.cpp | 9 --- 4 files changed, 133 insertions(+), 66 deletions(-) (limited to 'node') diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 0c061266..7f92d6ee 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -16,6 +16,18 @@ * along with this program. If not, see . */ +#include +#include +#include +#ifndef __WINDOWS__ +#include +#include +#include +#include +#include +#include +#endif + #include "JSONDB.hpp" #define ZT_JSONDB_HTTP_TIMEOUT 60000 @@ -27,9 +39,12 @@ static const std::map _ZT_JSONDB_GET_HEADERS; JSONDB::JSONDB(const std::string &basePath) : _basePath(basePath), + _rawInput(-1), + _rawOutput(-1), _summaryThreadRun(true) { if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { + // If base path is http:// we run in HTTP mode // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. // Typically it's just used with 127.0.0.1 anyway. std::string hn = _basePath.substr(7); @@ -46,16 +61,27 @@ JSONDB::JSONDB(const std::string &basePath) : _basePath = "/"; if (_basePath[0] != '/') _basePath = std::string("/") + _basePath; +#ifndef __WINDOWS__ + } else if (_basePath == "-") { + // If base path is "-" we run in stdin/stdout mode and expect our database to be populated on startup via stdin + // Not supported on Windows + _rawInput = STDIN_FILENO; + _rawOutput = STDOUT_FILENO; + fcntl(_rawInput,F_SETFL,O_NONBLOCK); +#endif } else { + // Default mode of operation is to store files in the filesystem OSUtils::mkdir(_basePath.c_str()); OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions } - unsigned int cnt = 0; - while (!_load(_basePath)) { - if ((++cnt & 7) == 0) - fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); - Thread::sleep(250); + if (_rawInput < 0) { + unsigned int cnt = 0; + while (!_load(_basePath)) { + if ((++cnt & 7) == 0) + fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); + Thread::sleep(250); + } } for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) @@ -89,7 +115,18 @@ JSONDB::~JSONDB() bool JSONDB::writeRaw(const std::string &n,const std::string &obj) { - if (_httpAddr) { + if (_rawOutput >= 0) { +#ifndef __WINDOWS__ + if (obj.length() > 0) { + Mutex::Lock _l(_rawLock); + if (write(_rawOutput,obj.c_str(),obj.length() + 1) > 0) + return true; + } else { + return true; + } +#endif + return false; + } else if (_httpAddr) { std::map headers; std::string body; std::map reqHeaders; @@ -205,11 +242,13 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); - if (_httpAddr) { - // Deletion is currently done by Central in harnessed mode - //std::map headers; - //std::string body; - //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + if (_rawOutput >= 0) { + // In harnessed mode, deletes occur in Central or other management + // software and do not need to be executed this way. + } else if (_httpAddr) { + std::map headers; + std::string body; + Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); if (path.length()) @@ -232,11 +271,13 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ char n[256]; OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); - if (_httpAddr) { - // Deletion is currently done by the caller in Central harnessed mode - //std::map headers; - //std::string body; - //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + if (_rawOutput >= 0) { + // In harnessed mode, deletes occur in Central or other management + // software and do not need to be executed this way. + } else if (_httpAddr) { + std::map headers; + std::string body; + Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); } else { const std::string path(_genPath(n,false)); if (path.length()) @@ -263,9 +304,41 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ void JSONDB::threadMain() throw() { +#ifndef __WINDOWS__ + fd_set readfds,nullfds; + char *const readbuf = (_rawInput >= 0) ? (new char[1048576]) : (char *)0; + std::string rawInputBuf; + FD_ZERO(&readfds); + FD_ZERO(&nullfds); +#endif + std::vector todo; + while (_summaryThreadRun) { - Thread::sleep(10); +#ifndef __WINDOWS__ + if (_rawInput < 0) { + Thread::sleep(25); + } else { + FD_SET(_rawInput,&readfds); + struct timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 25000; + select(_rawInput+1,&readfds,&nullfds,&nullfds,&tv); + if (FD_ISSET(_rawInput,&readfds)) { + const long rn = (long)read(_rawInput,readbuf,1048576); + for(long i=0;i 0) { + _add(OSUtils::jsonParse(rawInputBuf)); + rawInputBuf.clear(); + } + } + } + } +#else + Thread::sleep(25); +#endif { Mutex::Lock _l(_summaryThread_m); @@ -273,7 +346,6 @@ void JSONDB::threadMain() continue; else _summaryThreadToDo.swap(todo); } - const uint64_t now = OSUtils::now(); for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { const uint64_t networkId = *ii; @@ -340,10 +412,46 @@ void JSONDB::threadMain() todo.clear(); } + +#ifndef __WINDOWS__ + delete [] readbuf; +#endif +} + +bool JSONDB::_add(const nlohmann::json &j) +{ + try { + if (j.is_object()) { + std::string id(OSUtils::jsonString(j["id"],"0")); + std::string objtype(OSUtils::jsonString(j["objtype"],"")); + + if ((id.length() == 16)&&(objtype == "network")) { + const uint64_t nwid = Utils::hexStrToU64(id.c_str()); + if (nwid) { + Mutex::Lock _l(_networks_m); + _networks[nwid].config = nlohmann::json::to_msgpack(j); + return true; + } + } else if ((id.length() == 10)&&(objtype == "member")) { + const uint64_t mid = Utils::hexStrToU64(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + if ((mid)&&(nwid)) { + Mutex::Lock _l(_networks_m); + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); + _members[mid].insert(nwid); + return true; + } + } + } + } catch ( ... ) {} + return false; } bool JSONDB::_load(const std::string &p) { + // This is not used in stdin/stdout mode. Instead data is populated by + // sending it all to stdin. + if (_httpAddr) { // In HTTP harnessed mode we download our entire working data set on startup. @@ -357,24 +465,9 @@ bool JSONDB::_load(const std::string &p) if (dbImg.is_object()) { Mutex::Lock _l(_networks_m); for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { - nlohmann::json &j = i.value(); - if (j.is_object()) { - std::string id(OSUtils::jsonString(j["id"],"0")); - std::string objtype(OSUtils::jsonString(j["objtype"],"")); - - if ((id.length() == 16)&&(objtype == "network")) { - const uint64_t nwid = Utils::hexStrToU64(id.c_str()); - if (nwid) - _networks[nwid].config = nlohmann::json::to_msgpack(j); - } else if ((id.length() == 10)&&(objtype == "member")) { - const uint64_t mid = Utils::hexStrToU64(id.c_str()); - const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); - if ((mid)&&(nwid)) { - _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); - _members[mid].insert(nwid); - } - } - } + try { + _add(i.value()); + } catch ( ... ) {} } return true; } @@ -391,25 +484,7 @@ bool JSONDB::_load(const std::string &p) std::string buf; if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { try { - nlohmann::json j(OSUtils::jsonParse(buf)); - std::string id(OSUtils::jsonString(j["id"],"0")); - std::string objtype(OSUtils::jsonString(j["objtype"],"")); - - if ((id.length() == 16)&&(objtype == "network")) { - const uint64_t nwid = Utils::hexStrToU64(id.c_str()); - if (nwid) { - Mutex::Lock _l(_networks_m); - _networks[nwid].config = nlohmann::json::to_msgpack(j); - } - } else if ((id.length() == 10)&&(objtype == "member")) { - const uint64_t mid = Utils::hexStrToU64(id.c_str()); - const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); - if ((mid)&&(nwid)) { - Mutex::Lock _l(_networks_m); - _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); - _members[mid].insert(nwid); - } - } + _add(OSUtils::jsonParse(buf)); } catch ( ... ) {} } } else { diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 99b69ba2..23d00a51 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -145,12 +145,15 @@ public: throw(); private: + bool _add(const nlohmann::json &j); bool _load(const std::string &p); void _recomputeSummaryInfo(const uint64_t networkId); std::string _genPath(const std::string &n,bool create); std::string _basePath; InetAddress _httpAddr; + int _rawInput,_rawOutput; + Mutex _rawLock; Thread _summaryThread; std::vector _summaryThreadToDo; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 51955bf3..e5e10476 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -81,14 +81,12 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) if (peer) { if (!trusted) { if (!dearmor(peer->key())) { - //fprintf(stderr,"dropped packet from %s(%s), MAC authentication failed (size: %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size()); RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops()); return true; } } if (!uncompress()) { - //fprintf(stderr,"dropped packet from %s(%s), compressed data invalid (size %u, verb may be %u)" ZT_EOL_S,sourceAddress.toString().c_str(),_path->address().toString().c_str(),size(),(unsigned int)verb()); RR->t->incomingPacketInvalid(tPtr,_path,packetId(),sourceAddress,hops(),Packet::VERB_NOP,"LZ4 decompression failed"); return true; } diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 57ecce78..11005945 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -243,7 +243,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } } @@ -258,7 +257,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; idx |= (unsigned long)*(reinterpret_cast(data) + 20); - //printf("<< GET_DATA @%u from %.10llx for %s\n",(unsigned int)idx,origin,Utils::hex(reinterpret_cast(data) + 1,16).c_str()); std::map< Array,_D >::iterator d(_dist.find(Array(reinterpret_cast(data) + 1))); if ((d != _dist.end())&&(idx < (unsigned long)d->second.bin.length())) { Buffer buf; @@ -267,7 +265,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void buf.append((uint32_t)idx); buf.append(d->second.bin.data() + idx,std::min((unsigned long)ZT_SOFTWARE_UPDATE_CHUNK_SIZE,(unsigned long)(d->second.bin.length() - idx))); _node.sendUserMessage((void *)0,origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); - //printf(">> DATA @%u\n",(unsigned int)idx); } } break; @@ -278,7 +275,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; idx |= (unsigned long)*(reinterpret_cast(data) + 20); - //printf("<< DATA @%u / %u bytes (we now have %u bytes)\n",(unsigned int)idx,(unsigned int)(len - 21),(unsigned int)_download.length()); if (idx == (unsigned long)_download.length()) { _download.append(reinterpret_cast(data) + 21,len - 21); if (_download.length() < _downloadLength) { @@ -287,7 +283,6 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } } @@ -334,7 +329,6 @@ bool SoftwareUpdater::check(const uint64_t now) (int)ZT_VENDOR_ZEROTIER, _channel.c_str()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); - //printf(">> GET_LATEST\n"); } if (_latestValid) @@ -360,7 +354,6 @@ bool SoftwareUpdater::check(const uint64_t now) if (OSUtils::writeFile(binPath.c_str(),_download)) { OSUtils::lockDownFile(binPath.c_str(),false); _latestValid = true; - //printf("VALID UPDATE\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); _download = std::string(); _downloadLength = 0; return true; @@ -370,7 +363,6 @@ bool SoftwareUpdater::check(const uint64_t now) } catch ( ... ) {} // any exception equals verification failure // If we get here, checks failed. - //printf("INVALID UPDATE (!!!)\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); OSUtils::rm(binPath.c_str()); _latestMeta = nlohmann::json(); _latestValid = false; @@ -382,7 +374,6 @@ bool SoftwareUpdater::check(const uint64_t now) gd.append(_downloadHashPrefix.data,16); gd.append((uint32_t)_download.length()); _node.sendUserMessage((void *)0,ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - //printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } -- cgit v1.2.3 From 7e6598e9ca28da7047176907f5cacbc53ab60afe Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 7 Aug 2017 14:13:08 -0700 Subject: Possible deadlock fix. --- node/Node.cpp | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index f3339068..0df3a97a 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -250,20 +250,23 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint _lastPingCheck = now; // Get networks that need config without leaving mutex locked - std::vector< SharedPtr > needConfig; { - Mutex::Lock _l(_networks_m); - Hashtable< uint64_t,SharedPtr >::Iterator i(_networks); - uint64_t *k = (uint64_t *)0; - SharedPtr *v = (SharedPtr *)0; - while (i.next(k,v)) { - if (((now - (*v)->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!(*v)->hasConfig())) - needConfig.push_back(*v); - (*v)->sendUpdatesToMembers(tptr); + std::vector< std::pair< SharedPtr,bool > > nwl; + { + Mutex::Lock _l(_networks_m); + nwl.reserve(_networks.size()+1); + Hashtable< uint64_t,SharedPtr >::Iterator i(_networks); + uint64_t *k = (uint64_t *)0; + SharedPtr *v = (SharedPtr *)0; + while (i.next(k,v)) + nwl.push_back( std::pair< SharedPtr,bool >(*v,(((now - (*v)->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!(*v)->hasConfig()))) ); + } + for(std::vector< std::pair< SharedPtr,bool > >::const_iterator n(nwl.begin());n!=nwl.end();++n) { + if (n->second) + n->first->requestConfiguration(tptr); + n->first->sendUpdatesToMembers(tptr); } } - for(std::vector< SharedPtr >::const_iterator n(needConfig.begin());n!=needConfig.end();++n) - (*n)->requestConfiguration(tptr); // Do pings and keepalives Hashtable< Address,std::vector > upstreamsToContact; -- cgit v1.2.3 From e3cf7567856267bc4f4af0f1fb857cab105602e8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 8 Aug 2017 13:21:10 -0700 Subject: Make rxQueue lock-free using an atomic counter ring buffer. --- node/AtomicCounter.hpp | 9 +++++++++ node/Switch.cpp | 36 +++++++++++------------------------- node/Switch.hpp | 27 +++++++++++++-------------- 3 files changed, 33 insertions(+), 39 deletions(-) (limited to 'node') diff --git a/node/AtomicCounter.hpp b/node/AtomicCounter.hpp index e1864db8..abb342fe 100644 --- a/node/AtomicCounter.hpp +++ b/node/AtomicCounter.hpp @@ -47,6 +47,15 @@ public: _v = 0; } + inline int load() const + { +#ifdef __GNUC__ + return __sync_or_and_fetch(&_v,0); +#else + return _v.load(); +#endif + } + inline int operator++() { #ifdef __GNUC__ diff --git a/node/Switch.cpp b/node/Switch.cpp index 9c9daac9..c509ef16 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -120,10 +120,8 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre // Total fragments must be more than 1, otherwise why are we // seeing a Packet::Fragment? - Mutex::Lock _l(_rxQueue_m); - RXQueueEntry *const rq = _findRXQueueEntry(now,fragmentPacketId); - - if ((!rq->timestamp)||(rq->packetId != fragmentPacketId)) { + RXQueueEntry *const rq = _findRXQueueEntry(fragmentPacketId); + if (rq->packetId != fragmentPacketId) { // No packet found, so we received a fragment without its head. rq->timestamp = now; @@ -250,10 +248,8 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre ((uint64_t)reinterpret_cast(data)[7]) ); - Mutex::Lock _l(_rxQueue_m); - RXQueueEntry *const rq = _findRXQueueEntry(now,packetId); - - if ((!rq->timestamp)||(rq->packetId != packetId)) { + RXQueueEntry *const rq = _findRXQueueEntry(packetId); + if (rq->packetId != packetId) { // If we have no other fragments yet, create an entry and save the head rq->timestamp = now; @@ -286,14 +282,7 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre // Packet is unfragmented, so just process it IncomingPacket packet(data,len,path,now); if (!packet.tryDecode(RR,tPtr)) { - Mutex::Lock _l(_rxQueue_m); - RXQueueEntry *rq = &(_rxQueue[ZT_RX_QUEUE_SIZE - 1]); - unsigned long i = ZT_RX_QUEUE_SIZE - 1; - while ((i)&&(rq->timestamp)) { - RXQueueEntry *tmp = &(_rxQueue[--i]); - if (tmp->timestamp < rq->timestamp) - rq = tmp; - } + RXQueueEntry *const rq = _nextRXQueueEntry(); rq->timestamp = now; rq->packetId = packet.packetId(); rq->frag0 = packet; @@ -590,15 +579,12 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) _outstandingWhoisRequests.erase(peer->address()); } - { // finish processing any packets waiting on peer's public key / identity - Mutex::Lock _l(_rxQueue_m); - unsigned long i = ZT_RX_QUEUE_SIZE; - while (i) { - RXQueueEntry *rq = &(_rxQueue[--i]); - if ((rq->timestamp)&&(rq->complete)) { - if (rq->frag0.tryDecode(RR,tPtr)) - rq->timestamp = 0; - } + // finish processing any packets waiting on peer's public key / identity + for(unsigned int ptr=0;ptrtimestamp)&&(rq->complete)) { + if (rq->frag0.tryDecode(RR,tPtr)) + rq->timestamp = 0; } } diff --git a/node/Switch.hpp b/node/Switch.hpp index 346aaca3..114bc5e1 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -169,25 +169,24 @@ private: bool complete; // if true, packet is complete }; RXQueueEntry _rxQueue[ZT_RX_QUEUE_SIZE]; - Mutex _rxQueue_m; + AtomicCounter _rxQueuePtr; - /* Returns the matching or oldest entry. Caller must check timestamp and - * packet ID to determine which. */ - inline RXQueueEntry *_findRXQueueEntry(uint64_t now,uint64_t packetId) + // Returns matching or next available RX queue entry + inline RXQueueEntry *_findRXQueueEntry(uint64_t packetId) { - RXQueueEntry *rq; - RXQueueEntry *oldest = &(_rxQueue[ZT_RX_QUEUE_SIZE - 1]); - unsigned long i = ZT_RX_QUEUE_SIZE; - while (i) { - rq = &(_rxQueue[--i]); + unsigned int ptr = static_cast(_rxQueuePtr.load()); + for(unsigned int k=0;kpacketId == packetId)&&(rq->timestamp)) return rq; - if ((now - rq->timestamp) >= ZT_RX_QUEUE_EXPIRE) - rq->timestamp = 0; - if (rq->timestamp < oldest->timestamp) - oldest = rq; } - return oldest; + return &(_rxQueue[static_cast(++_rxQueuePtr) % ZT_RX_QUEUE_SIZE]); + } + + // Returns next RX queue entry in ring buffer and increments ring counter + inline RXQueueEntry *_nextRXQueueEntry() + { + return &(_rxQueue[static_cast(++_rxQueuePtr) % ZT_RX_QUEUE_SIZE]); } // ZeroTier-layer TX queue entry -- cgit v1.2.3 From ff5e22031aa005d4bcb0839797ac4044f6992e50 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 8 Aug 2017 13:24:37 -0700 Subject: Small fix: should expire packets to prevent repeated WHOISes. --- node/Switch.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index c509ef16..053f793e 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -580,10 +580,11 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) } // finish processing any packets waiting on peer's public key / identity + const uint64_t now = RR->node->now(); for(unsigned int ptr=0;ptrtimestamp)&&(rq->complete)) { - if (rq->frag0.tryDecode(RR,tPtr)) + if ((rq->frag0.tryDecode(RR,tPtr))||((now - rq->timestamp) > ZT_RECEIVE_QUEUE_TIMEOUT)) rq->timestamp = 0; } } -- cgit v1.2.3 From a4bc40542bf11a6688b5e0e8aa0e06a7bc4fe544 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 14 Aug 2017 11:43:39 -0700 Subject: GCC/G++ build fixes, GitHub issue #563 --- node/AtomicCounter.hpp | 2 +- node/Node.cpp | 2 +- osdep/Binder.hpp | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'node') diff --git a/node/AtomicCounter.hpp b/node/AtomicCounter.hpp index abb342fe..34b58e91 100644 --- a/node/AtomicCounter.hpp +++ b/node/AtomicCounter.hpp @@ -50,7 +50,7 @@ public: inline int load() const { #ifdef __GNUC__ - return __sync_or_and_fetch(&_v,0); + return __sync_or_and_fetch(const_cast(&_v),0); #else return _v.load(); #endif diff --git a/node/Node.cpp b/node/Node.cpp index 0df3a97a..366ddbf0 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -100,7 +100,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 } else { idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0; n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1); - if ((n > 0)&&(n < sizeof(RR->publicIdentityStr))&&(n < sizeof(tmp))) { + if ((n > 0)&&(n < (int)sizeof(RR->publicIdentityStr))&&(n < (int)sizeof(tmp))) { if (memcmp(tmp,RR->publicIdentityStr,n)) stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr)); } diff --git a/osdep/Binder.hpp b/osdep/Binder.hpp index 17a0fbf6..e3c2dc02 100644 --- a/osdep/Binder.hpp +++ b/osdep/Binder.hpp @@ -227,7 +227,7 @@ public: case InetAddress::IP_SCOPE_GLOBAL: case InetAddress::IP_SCOPE_SHARED: case InetAddress::IP_SCOPE_PRIVATE: - for(int x=0;x(ip,std::string(devname))); } @@ -268,7 +268,7 @@ public: case InetAddress::IP_SCOPE_GLOBAL: case InetAddress::IP_SCOPE_SHARED: case InetAddress::IP_SCOPE_PRIVATE: - for(int x=0;x(ip,ifname)); } @@ -302,7 +302,7 @@ public: case InetAddress::IP_SCOPE_GLOBAL: case InetAddress::IP_SCOPE_SHARED: case InetAddress::IP_SCOPE_PRIVATE: - for(int x=0;x(ip,std::string(ifa->ifa_name))); } -- cgit v1.2.3 From fcaf1d89c260943d3c9c4021b9ab6fe89c1c4de8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 18 Aug 2017 13:59:22 -0700 Subject: Get rid of some noisy remote traces that should not be needed. --- node/IncomingPacket.cpp | 13 ++++---- node/Membership.cpp | 3 -- node/Trace.cpp | 79 ++----------------------------------------------- node/Trace.hpp | 9 +----- 4 files changed, 10 insertions(+), 94 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index e5e10476..3788708d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -66,10 +66,9 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) // packets are dropped on the floor. const uint64_t tpid = trustedPathId(); if (RR->topology->shouldInboundPathBeTrusted(_path->address(),tpid)) { - RR->t->incomingPacketTrustedPath(tPtr,_path,packetId(),sourceAddress,tpid,true); trusted = true; } else { - RR->t->incomingPacketTrustedPath(tPtr,_path,packetId(),sourceAddress,tpid,false); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops(),"path not trusted"); return true; } } else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) { @@ -81,7 +80,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) if (peer) { if (!trusted) { if (!dearmor(peer->key())) { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops()); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops(),"invalid MAC"); return true; } } @@ -246,10 +245,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool outp.armor(key,true,_path->nextOutgoingCounter()); _path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now()); } else { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid MAC"); } } else { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid identity"); } return true; @@ -257,7 +256,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool // Identity is the same as the one we already have -- check packet integrity if (!dearmor(peer->key())) { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid MAC"); return true; } @@ -282,7 +281,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool // Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap) SharedPtr newPeer(new Peer(RR,RR->identity,id)); if (!dearmor(newPeer->key())) { - RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops()); + RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid MAC"); return true; } diff --git a/node/Membership.cpp b/node/Membership.cpp index a1453307..17de6554 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -147,7 +147,6 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme return ADD_REJECTED; case 0: _com = com; - RR->t->credentialAccepted(tPtr,com); return ADD_ACCEPTED_NEW; case 1: return ADD_DEFERRED_FOR_WHOIS; @@ -179,7 +178,6 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot RR->t->credentialRejected(tPtr,cred,"invalid"); return Membership::ADD_REJECTED; case 0: - RR->t->credentialAccepted(tPtr,cred); if (!rc) rc = &(remoteCreds[cred.id()]); *rc = cred; @@ -205,7 +203,6 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme switch(ct) { case Credential::CREDENTIAL_TYPE_COM: if (rev.threshold() > _comRevocationThreshold) { - RR->t->credentialAccepted(tPtr,rev); _comRevocationThreshold = rev.threshold(); return ADD_ACCEPTED_NEW; } diff --git a/node/Trace.cpp b/node/Trace.cpp index 98a4adcb..8e78b676 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -164,12 +164,7 @@ void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved) -{ - // TODO -} - -void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops) +void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason) { char tmp[128]; Dictionary d; @@ -179,6 +174,8 @@ void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const Sh d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source); d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp)); d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket()); + if (reason) + d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); _send(tPtr,d,0); } @@ -344,76 +341,6 @@ void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char * _send(tPtr,d,c.networkId()); } -void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,c.networkId()); -} - -void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,c.networkId()); -} - -void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); - _send(tPtr,d,0); -} - -void Trace::credentialAccepted(void *const tPtr,const Capability &c) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - _send(tPtr,d,c.networkId()); -} - -void Trace::credentialAccepted(void *const tPtr,const Tag &c) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value()); - _send(tPtr,d,c.networkId()); -} - -void Trace::credentialAccepted(void *const tPtr,const Revocation &c) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target()); - _send(tPtr,d,c.networkId()); -} - void Trace::_send(void *const tPtr,const Dictionary &d) { #ifdef ZT_TRACE diff --git a/node/Trace.hpp b/node/Trace.hpp index d66d0871..a7b2b194 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -108,8 +108,7 @@ public: void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId); void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath); - void incomingPacketTrustedPath(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved); - void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops); + void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason); void incomingPacketInvalid(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const Packet::Verb verb,const char *reason); void incomingPacketDroppedHELLO(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const char *reason); @@ -142,12 +141,6 @@ public: void credentialRejected(void *const tPtr,const Capability &c,const char *reason); void credentialRejected(void *const tPtr,const Tag &c,const char *reason); void credentialRejected(void *const tPtr,const Revocation &c,const char *reason); - void credentialAccepted(void *const tPtr,const CertificateOfMembership &c); - void credentialAccepted(void *const tPtr,const CertificateOfOwnership &c); - void credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c); - void credentialAccepted(void *const tPtr,const Capability &c); - void credentialAccepted(void *const tPtr,const Tag &c); - void credentialAccepted(void *const tPtr,const Revocation &c); private: const RuntimeEnvironment *const RR; -- cgit v1.2.3 From 64758c46b672f8b3182c370aed6b81a07780c093 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 13:40:51 -0700 Subject: Implement peer serialization and deserialization. --- node/Peer.cpp | 2 +- node/Peer.hpp | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++ node/Topology.cpp | 31 +++++++++++++++---- node/Topology.hpp | 2 ++ service/OneService.cpp | 25 ++++++++++++--- 5 files changed, 132 insertions(+), 12 deletions(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index 986e52ef..127f222c 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -369,7 +369,7 @@ void Peer::tryMemorizedPath(void *tPtr,uint64_t now) _lastTriedMemorizedPath = now; InetAddress mp; if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp)) - attemptToContactAt(tPtr,InetAddress(),mp,now,true,0); + attemptToContactAt(tPtr,-1,mp,now,true,0); } } diff --git a/node/Peer.hpp b/node/Peer.hpp index c6423a59..af9163a5 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -439,6 +439,90 @@ public: return false; } + /** + * Serialize a peer for storage in local cache + * + * This does not serialize everything, just identity and addresses where the peer + * may be reached. + */ + template + inline void serialize(Buffer &b) const + { + b.append((uint8_t)0); + + _id.serialize(b); + + b.append(_lastReceive); + b.append(_lastNontrivialReceive); + b.append(_lastTriedMemorizedPath); + b.append(_lastDirectPathPushSent); + b.append(_lastDirectPathPushReceive); + b.append(_lastCredentialRequestSent); + b.append(_lastWhoisRequestReceived); + b.append(_lastEchoRequestReceived); + b.append(_lastComRequestReceived); + b.append(_lastComRequestSent); + b.append(_lastCredentialsReceived); + b.append(_lastTrustEstablishedPacketReceived); + + b.append((uint16_t)_vProto); + b.append((uint16_t)_vMajor); + b.append((uint16_t)_vMinor); + b.append((uint16_t)_vRevision); + + { + Mutex::Lock _l(_paths_m); + unsigned int pcount = 0; + if (_v4Path.p) ++pcount; + if (_v6Path.p) ++pcount; + b.append((uint8_t)pcount); + if (_v4Path.p) _v4Path.p->address().serialize(b); + if (_v6Path.p) _v6Path.p->address().serialize(b); + } + + b.append((uint16_t)0); + } + + template + inline static SharedPtr deserializeFromCache(uint64_t now,void *tPtr,Buffer &b,const RuntimeEnvironment *renv) + { + try { + unsigned int ptr = 0; + if (b[ptr++] != 0) + return SharedPtr(); + + Identity id; + ptr += id.deserialize(b,ptr); + if (!id) + return SharedPtr(); + + SharedPtr p(new Peer(renv,renv->identity,id)); + + ptr += 12 * 8; // skip deserializing ephemeral state in this case + + p->_vProto = b.template at(ptr); ptr += 2; + p->_vMajor = b.template at(ptr); ptr += 2; + p->_vMinor = b.template at(ptr); ptr += 2; + p->_vRevision = b.template at(ptr); ptr += 2; + + const unsigned int pcount = (unsigned int)b[ptr++]; + for(unsigned int i=0;iattemptToContactAt(tPtr,-1,inaddr,now,true,0); + } catch ( ... ) { + break; + } + } + + return p; + } catch ( ... ) { + return SharedPtr(); + } + } + private: struct _PeerPath { diff --git a/node/Topology.cpp b/node/Topology.cpp index edca0180..aeca59a7 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -88,6 +88,15 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : addWorld(tPtr,defaultPlanet,false); } +Topology::~Topology() +{ + Hashtable< Address,SharedPtr >::Iterator i(_peers); + Address *a = (Address *)0; + SharedPtr *p = (SharedPtr *)0; + while (i.next(a,p)) + _savePeer((void *)0,*p); +} + SharedPtr Topology::addPeer(void *tPtr,const SharedPtr &peer) { SharedPtr np; @@ -113,23 +122,21 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) return *ap; } - /* try { - char buf[ZT_PEER_MAX_SERIALIZED_STATE_SIZE]; + Buffer buf; uint64_t idbuf[2]; idbuf[0] = zta.toInt(); idbuf[1] = 0; - int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER,idbuf,buf,(unsigned int)sizeof(buf)); + int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER,idbuf,buf.unsafeData(),ZT_PEER_MAX_SERIALIZED_STATE_SIZE); if (len > 0) { Mutex::Lock _l(_peers_m); SharedPtr &ap = _peers[zta]; if (ap) return ap; - ap = Peer::createFromStateUpdate(RR,tPtr,buf,len); + ap = Peer::deserializeFromCache(RR->node->now(),tPtr,buf,RR); if (!ap) _peers.erase(zta); return ap; } } catch ( ... ) {} // ignore invalid identities or other strage failures - */ return SharedPtr(); } @@ -383,8 +390,10 @@ void Topology::doPeriodicTasks(void *tPtr,uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) + if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) { + _savePeer(tPtr,*p); _peers.erase(*a); + } } } @@ -440,4 +449,14 @@ void Topology::_memoizeUpstreams(void *tPtr) _cor.sign(RR->identity,RR->node->now()); } +void Topology::_savePeer(void *tPtr,const SharedPtr &peer) +{ + try { + Buffer buf; + peer->serialize(buf); + uint64_t tmpid[2]; tmpid[0] = peer->address().toInt(); tmpid[1] = 0; + RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER,tmpid,buf.data(),buf.size()); + } catch ( ... ) {} // sanity check, discard invalid entries +} + } // namespace ZeroTier diff --git a/node/Topology.hpp b/node/Topology.hpp index 30e58abc..04dfb1cc 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -59,6 +59,7 @@ class Topology { public: Topology(const RuntimeEnvironment *renv,void *tPtr); + ~Topology(); /** * Add a peer to database @@ -419,6 +420,7 @@ public: private: Identity _getIdentity(void *tPtr,const Address &zta); void _memoizeUpstreams(void *tPtr); + void _savePeer(void *tPtr,const SharedPtr &peer); const RuntimeEnvironment *const RR; diff --git a/service/OneService.cpp b/service/OneService.cpp index f5f8700a..cd33e399 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -2047,6 +2047,8 @@ public: char p[1024]; FILE *f; bool secure = false; + char dirname[1024]; + dirname[0] = 0; switch(type) { case ZT_STATE_OBJECT_IDENTITY_PUBLIC: @@ -2060,12 +2062,18 @@ public: OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]); + OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "moons.d",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.moon",dirname,(unsigned long long)id[0]); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]); + OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "networks.d",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.conf",dirname,(unsigned long long)id[0]); secure = true; break; + case ZT_STATE_OBJECT_PEER: + OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "peers.d",_homePath.c_str()); + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.10llx.peer",dirname,(unsigned long long)id[0]); + break; default: return; } @@ -2084,6 +2092,10 @@ public: } f = fopen(p,"w"); + if ((!f)&&(dirname[0])) { // create subdirectory if it does not exist + OSUtils::mkdir(dirname); + f = fopen(p,"w"); + } if (f) { if (fwrite(data,len,1,f) != 1) fprintf(stderr,"WARNING: unable to write to file: %s (I/O error)" ZT_EOL_S,p); @@ -2108,15 +2120,18 @@ public: case ZT_STATE_OBJECT_IDENTITY_SECRET: OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str()); break; - case ZT_STATE_OBJECT_NETWORK_CONFIG: - OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); - break; case ZT_STATE_OBJECT_PLANET: OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id); break; + case ZT_STATE_OBJECT_NETWORK_CONFIG: + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id); + break; + case ZT_STATE_OBJECT_PEER: + OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "peers.d/%.10llx.conf",_homePath.c_str(),(unsigned long long)id[0]); + break; default: return -1; } -- cgit v1.2.3 From 9cfc10952748487bed318ec11221eeccbb285c89 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 14:00:08 -0700 Subject: Tighten a few timings. --- node/Constants.hpp | 4 ++-- node/Switch.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 3f050ead..27dce075 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -226,12 +226,12 @@ /** * Delay between WHOIS retries in ms */ -#define ZT_WHOIS_RETRY_DELAY 1000 +#define ZT_WHOIS_RETRY_DELAY 500 /** * Maximum identity WHOIS retries (each attempt tries consulting a different peer) */ -#define ZT_MAX_WHOIS_RETRIES 4 +#define ZT_MAX_WHOIS_RETRIES 5 /** * Transmit queue entry timeout diff --git a/node/Switch.cpp b/node/Switch.cpp index 053f793e..0d39bee9 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -630,9 +630,9 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) { // Time out TX queue packets that never got WHOIS lookups or other info. Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { - if (_trySend(tPtr,txi->packet,txi->encrypt)) + if (_trySend(tPtr,txi->packet,txi->encrypt)) { _txQueue.erase(txi++); - else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { + } else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { RR->t->txTimedOut(tPtr,txi->dest); _txQueue.erase(txi++); } else ++txi; -- cgit v1.2.3 From a156a4dbe2ffe5fc5f870144fe80efc172901d8e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 15:12:00 -0700 Subject: Symmetric NAT cleanup. --- node/SelfAwareness.cpp | 52 ++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 23 deletions(-) (limited to 'node') diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index cdbb6303..0af0d691 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -147,13 +147,14 @@ std::vector SelfAwareness::getSymmetricNatPredictions() * read or modify traffic, but they could gather meta-data for forensics * purpsoes or use this as a DOS attack vector. */ - std::map< uint32_t,std::pair > maxPortByIp; + std::map< uint32_t,unsigned int > maxPortByIp; InetAddress theOneTrueSurface; - bool symmetric = false; { Mutex::Lock _l(_phy_m); - { // First get IPs from only trusted peers, and perform basic NAT type characterization + // First check to see if this is a symmetric NAT and enumerate external IPs learned from trusted peers + bool symmetric = false; + { Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); PhySurfaceKey *k = (PhySurfaceKey *)0; PhySurfaceEntry *e = (PhySurfaceEntry *)0; @@ -163,42 +164,47 @@ std::vector SelfAwareness::getSymmetricNatPredictions() theOneTrueSurface = e->mySurface; else if (theOneTrueSurface != e->mySurface) symmetric = true; - maxPortByIp[reinterpret_cast(&(e->mySurface))->sin_addr.s_addr] = std::pair(e->ts,e->mySurface.port()); + maxPortByIp[reinterpret_cast(&(e->mySurface))->sin_addr.s_addr] = e->mySurface.port(); } } } + if (!symmetric) + return std::vector(); - { // Then find max port per IP from a trusted peer + { // Then find the highest issued port per IP Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); PhySurfaceKey *k = (PhySurfaceKey *)0; PhySurfaceEntry *e = (PhySurfaceEntry *)0; while (i.next(k,e)) { if ((e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { - std::map< uint32_t,std::pair >::iterator mp(maxPortByIp.find(reinterpret_cast(&(e->mySurface))->sin_addr.s_addr)); - if ((mp != maxPortByIp.end())&&(mp->second.first < e->ts)) { - mp->second.first = e->ts; - mp->second.second = e->mySurface.port(); - } + const unsigned int port = e->mySurface.port(); + std::map< uint32_t,unsigned int >::iterator mp(maxPortByIp.find(reinterpret_cast(&(e->mySurface))->sin_addr.s_addr)); + if ((mp != maxPortByIp.end())&&(mp->second < port)) + mp->second = port; } } } } - if (symmetric) { - std::vector r; - for(unsigned int k=1;k<=3;++k) { - for(std::map< uint32_t,std::pair >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) { - unsigned int p = i->second.second + k; - if (p > 65535) p -= 64511; - InetAddress pred(&(i->first),4,p); - if (std::find(r.begin(),r.end(),pred) == r.end()) - r.push_back(pred); - } - } - return r; + std::vector r; + + // Try next port up from max for each + for(std::map< uint32_t,unsigned int >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) { + unsigned int p = i->second + 1; + if (p > 65535) p -= 64511; + const InetAddress pred(&(i->first),4,p); + if (std::find(r.begin(),r.end(),pred) == r.end()) + r.push_back(pred); + } + + // Try a random port for each -- there are only 65535 so eventually it should work + for(std::map< uint32_t,unsigned int >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) { + const InetAddress pred(&(i->first),4,1024 + ((unsigned int)RR->node->prng() % 64511)); + if (std::find(r.begin(),r.end(),pred) == r.end()) + r.push_back(pred); } - return std::vector(); + return r; } } // namespace ZeroTier -- cgit v1.2.3 From b1d94c9f9324a31887dc6edc99ed58d4f9b187db Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 15:19:26 -0700 Subject: Performance improvement to RX queue ring buffer. --- node/Switch.hpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/node/Switch.hpp b/node/Switch.hpp index 114bc5e1..88415541 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -174,19 +174,20 @@ private: // Returns matching or next available RX queue entry inline RXQueueEntry *_findRXQueueEntry(uint64_t packetId) { - unsigned int ptr = static_cast(_rxQueuePtr.load()); - for(unsigned int k=0;k(_rxQueuePtr.load()); + for(unsigned int k=1;k<=ZT_RX_QUEUE_SIZE;++k) { + RXQueueEntry *rq = &(_rxQueue[(current - k) % ZT_RX_QUEUE_SIZE]); if ((rq->packetId == packetId)&&(rq->timestamp)) return rq; } - return &(_rxQueue[static_cast(++_rxQueuePtr) % ZT_RX_QUEUE_SIZE]); + ++_rxQueuePtr; + return &(_rxQueue[static_cast(current) % ZT_RX_QUEUE_SIZE]); } - // Returns next RX queue entry in ring buffer and increments ring counter + // Returns current entry in rx queue ring buffer and increments ring pointer inline RXQueueEntry *_nextRXQueueEntry() { - return &(_rxQueue[static_cast(++_rxQueuePtr) % ZT_RX_QUEUE_SIZE]); + return &(_rxQueue[static_cast((++_rxQueuePtr) - 1) % ZT_RX_QUEUE_SIZE]); } // ZeroTier-layer TX queue entry -- cgit v1.2.3 From 6ee201865b12f5b0f16208f6d696b1bf00197eaf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 16:42:17 -0700 Subject: Clean up WHOIS code. --- node/Capability.cpp | 3 +- node/CertificateOfMembership.cpp | 3 +- node/CertificateOfOwnership.cpp | 3 +- node/Constants.hpp | 9 +-- node/IncomingPacket.cpp | 4 +- node/Node.cpp | 26 ++++---- node/Revocation.cpp | 3 +- node/Switch.cpp | 132 ++++++++++++++++++++------------------- node/Switch.hpp | 24 +++---- node/Tag.cpp | 3 +- node/Topology.cpp | 35 +++-------- node/Topology.hpp | 14 +---- 12 files changed, 115 insertions(+), 144 deletions(-) (limited to 'node') diff --git a/node/Capability.cpp b/node/Capability.cpp index 0e02025a..47dca1fc 100644 --- a/node/Capability.cpp +++ b/node/Capability.cpp @@ -30,6 +30,7 @@ #include "Topology.hpp" #include "Switch.hpp" #include "Network.hpp" +#include "Node.hpp" namespace ZeroTier { @@ -59,7 +60,7 @@ int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const if (!id.verify(tmp.data(),tmp.size(),_custody[c].signature)) return -1; } else { - RR->sw->requestWhois(tPtr,_custody[c].from); + RR->sw->requestWhois(tPtr,RR->node->now(),_custody[c].from); return 1; } } diff --git a/node/CertificateOfMembership.cpp b/node/CertificateOfMembership.cpp index 100253e1..dedcccff 100644 --- a/node/CertificateOfMembership.cpp +++ b/node/CertificateOfMembership.cpp @@ -29,6 +29,7 @@ #include "Topology.hpp" #include "Switch.hpp" #include "Network.hpp" +#include "Node.hpp" namespace ZeroTier { @@ -223,7 +224,7 @@ int CertificateOfMembership::verify(const RuntimeEnvironment *RR,void *tPtr) con const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(tPtr,_signedBy); + RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy); return 1; } diff --git a/node/CertificateOfOwnership.cpp b/node/CertificateOfOwnership.cpp index 31d0ae18..eeb0d99c 100644 --- a/node/CertificateOfOwnership.cpp +++ b/node/CertificateOfOwnership.cpp @@ -30,6 +30,7 @@ #include "Topology.hpp" #include "Switch.hpp" #include "Network.hpp" +#include "Node.hpp" namespace ZeroTier { @@ -39,7 +40,7 @@ int CertificateOfOwnership::verify(const RuntimeEnvironment *RR,void *tPtr) cons return -1; const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(tPtr,_signedBy); + RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy); return 1; } try { diff --git a/node/Constants.hpp b/node/Constants.hpp index 27dce075..cda1af3b 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -228,20 +228,15 @@ */ #define ZT_WHOIS_RETRY_DELAY 500 -/** - * Maximum identity WHOIS retries (each attempt tries consulting a different peer) - */ -#define ZT_MAX_WHOIS_RETRIES 5 - /** * Transmit queue entry timeout */ -#define ZT_TRANSMIT_QUEUE_TIMEOUT (ZT_WHOIS_RETRY_DELAY * (ZT_MAX_WHOIS_RETRIES + 1)) +#define ZT_TRANSMIT_QUEUE_TIMEOUT 5000 /** * Receive queue entry timeout */ -#define ZT_RECEIVE_QUEUE_TIMEOUT (ZT_WHOIS_RETRY_DELAY * (ZT_MAX_WHOIS_RETRIES + 1)) +#define ZT_RECEIVE_QUEUE_TIMEOUT 5000 /** * Maximum latency to allow for OK(HELLO) before packet is discarded diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 3788708d..685f2f09 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -115,7 +115,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr) case Packet::VERB_REMOTE_TRACE: return _doREMOTE_TRACE(RR,tPtr,peer); } } else { - RR->sw->requestWhois(tPtr,sourceAddress); + RR->sw->requestWhois(tPtr,RR->node->now(),sourceAddress); return false; } } catch ( ... ) { @@ -556,7 +556,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar ++count; } else { // Request unknown WHOIS from upstream from us (if we have one) - RR->sw->requestWhois(tPtr,addr); + RR->sw->requestWhois(tPtr,RR->node->now(),addr); } } diff --git a/node/Node.cpp b/node/Node.cpp index 366ddbf0..09260172 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -249,6 +249,19 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint try { _lastPingCheck = now; + // Do pings and keepalives + Hashtable< Address,std::vector > upstreamsToContact; + RR->topology->getUpstreamsToContact(upstreamsToContact); + _PingPeersThatNeedPing pfunc(RR,tptr,upstreamsToContact,now); + RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); + + // Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds) + Hashtable< Address,std::vector >::Iterator i(upstreamsToContact); + Address *upstreamAddress = (Address *)0; + std::vector *upstreamStableEndpoints = (std::vector *)0; + while (i.next(upstreamAddress,upstreamStableEndpoints)) + RR->sw->requestWhois(tptr,now,*upstreamAddress); + // Get networks that need config without leaving mutex locked { std::vector< std::pair< SharedPtr,bool > > nwl; @@ -268,19 +281,6 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint } } - // Do pings and keepalives - Hashtable< Address,std::vector > upstreamsToContact; - RR->topology->getUpstreamsToContact(upstreamsToContact); - _PingPeersThatNeedPing pfunc(RR,tptr,upstreamsToContact,now); - RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc); - - // Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds) - Hashtable< Address,std::vector >::Iterator i(upstreamsToContact); - Address *upstreamAddress = (Address *)0; - std::vector *upstreamStableEndpoints = (std::vector *)0; - while (i.next(upstreamAddress,upstreamStableEndpoints)) - RR->sw->requestWhois(tptr,*upstreamAddress); - // Update online status, post status change as event const bool oldOnline = _online; _online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot())); diff --git a/node/Revocation.cpp b/node/Revocation.cpp index 026058da..89a2db95 100644 --- a/node/Revocation.cpp +++ b/node/Revocation.cpp @@ -30,6 +30,7 @@ #include "Topology.hpp" #include "Switch.hpp" #include "Network.hpp" +#include "Node.hpp" namespace ZeroTier { @@ -39,7 +40,7 @@ int Revocation::verify(const RuntimeEnvironment *RR,void *tPtr) const return -1; const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(tPtr,_signedBy); + RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy); return 1; } try { diff --git a/node/Switch.cpp b/node/Switch.cpp index 0d39bee9..8446602c 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -50,7 +50,6 @@ namespace ZeroTier { Switch::Switch(const RuntimeEnvironment *renv) : RR(renv), _lastBeaconResponse(0), - _outstandingWhoisRequests(32), _lastUniteAttempt(8) // only really used on root servers and upstreams, and it'll grow there just fine { } @@ -229,8 +228,8 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre } } } else { - relayTo = RR->topology->getUpstreamPeer(&source,1,true); - if (relayTo) + relayTo = RR->topology->getUpstreamPeer(); + if ((relayTo)&&(relayTo->address() != source)) relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true); } } @@ -553,33 +552,35 @@ void Switch::send(void *tPtr,Packet &packet,bool encrypt) } } -void Switch::requestWhois(void *tPtr,const Address &addr) +void Switch::requestWhois(void *tPtr,const uint64_t now,const Address &addr) { if (addr == RR->identity.address()) return; - bool inserted = false; + { - Mutex::Lock _l(_outstandingWhoisRequests_m); - WhoisRequest &r = _outstandingWhoisRequests[addr]; - if (r.lastSent) { - r.retries = 0; // reset retry count if entry already existed, but keep waiting and retry again after normal timeout - } else { - r.lastSent = RR->node->now(); - inserted = true; - } + Mutex::Lock _l(_lastSentWhoisRequest_m); + uint64_t &last = _lastSentWhoisRequest[addr]; + if ((now - last) < ZT_WHOIS_RETRY_DELAY) + return; + else last = now; + } + + const SharedPtr upstream(RR->topology->getUpstreamPeer()); + if (upstream) { + Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); + addr.appendTo(outp); + RR->node->expectReplyTo(outp.packetId()); + send(tPtr,outp,true); } - if (inserted) - _sendWhoisRequest(tPtr,addr,(const Address *)0,0); } void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) { - { // cancel pending WHOIS since we now know this peer - Mutex::Lock _l(_outstandingWhoisRequests_m); - _outstandingWhoisRequests.erase(peer->address()); + { + Mutex::Lock _l(_lastSentWhoisRequest_m); + _lastSentWhoisRequest.erase(peer->address()); } - // finish processing any packets waiting on peer's public key / identity const uint64_t now = RR->node->now(); for(unsigned int ptr=0;ptr &peer) } } - { // finish sending any packets waiting on peer's public key / identity + { Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { if (txi->dest == peer->address()) { - if (_trySend(tPtr,txi->packet,txi->encrypt)) + if (_trySend(tPtr,txi->packet,txi->encrypt)) { _txQueue.erase(txi++); - else ++txi; - } else ++txi; - } - } -} - -unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) -{ - unsigned long nextDelay = 0xffffffff; // ceiling delay, caller will cap to minimum - - { // Retry outstanding WHOIS requests - Mutex::Lock _l(_outstandingWhoisRequests_m); - Hashtable< Address,WhoisRequest >::Iterator i(_outstandingWhoisRequests); - Address *a = (Address *)0; - WhoisRequest *r = (WhoisRequest *)0; - while (i.next(a,r)) { - const unsigned long since = (unsigned long)(now - r->lastSent); - if (since >= ZT_WHOIS_RETRY_DELAY) { - if (r->retries >= ZT_MAX_WHOIS_RETRIES) { - _outstandingWhoisRequests.erase(*a); } else { - r->lastSent = now; - r->peersConsulted[r->retries] = _sendWhoisRequest(tPtr,*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0); - ++r->retries; - nextDelay = std::min(nextDelay,(unsigned long)ZT_WHOIS_RETRY_DELAY); + ++txi; } } else { - nextDelay = std::min(nextDelay,ZT_WHOIS_RETRY_DELAY - since); + ++txi; } } } +} - { // Time out TX queue packets that never got WHOIS lookups or other info. +unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) +{ + const uint64_t timeSinceLastCheck = now - _lastCheckedQueues; + if (timeSinceLastCheck < ZT_WHOIS_RETRY_DELAY) + return (unsigned long)(ZT_WHOIS_RETRY_DELAY - timeSinceLastCheck); + _lastCheckedQueues = now; + + { Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { if (_trySend(tPtr,txi->packet,txi->encrypt)) { _txQueue.erase(txi++); } else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { RR->t->txTimedOut(tPtr,txi->dest); - _txQueue.erase(txi++); - } else ++txi; + _txQueue.erase(txi); + ++txi; + } else if (!RR->topology->getPeer(tPtr,txi->dest)) { + requestWhois(tPtr,now,txi->dest); + ++txi; + } else { + ++txi; + } + } + } + + for(unsigned int ptr=0;ptrtimestamp)&&(rq->complete)) { + if ((rq->frag0.tryDecode(RR,tPtr))||((now - rq->timestamp) > ZT_RECEIVE_QUEUE_TIMEOUT)) { + rq->timestamp = 0; + } else { + const Address src(rq->frag0.source()); + if (!RR->topology->getPeer(tPtr,src)) + requestWhois(tPtr,now,src); + } } } - { // Remove really old last unite attempt entries to keep table size controlled + { Mutex::Lock _l(_lastUniteAttempt_m); Hashtable< _LastUniteKey,uint64_t >::Iterator i(_lastUniteAttempt); _LastUniteKey *k = (_LastUniteKey *)0; @@ -650,7 +655,18 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) } } - return nextDelay; + { + Mutex::Lock _l(_lastSentWhoisRequest_m); + Hashtable< Address,uint64_t >::Iterator i(_lastSentWhoisRequest); + Address *a = (Address *)0; + uint64_t *ts = (uint64_t *)0; + while (i.next(a,ts)) { + if ((now - *ts) > (ZT_WHOIS_RETRY_DELAY * 2)) + _lastSentWhoisRequest.erase(*a); + } + } + + return ZT_WHOIS_RETRY_DELAY; } bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address &destination) @@ -664,18 +680,6 @@ bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address return false; } -Address Switch::_sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted) -{ - SharedPtr upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false)); - if (upstream) { - Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS); - addr.appendTo(outp); - RR->node->expectReplyTo(outp.packetId()); - send(tPtr,outp,true); - } - return Address(); -} - bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) { SharedPtr viaPath; @@ -709,7 +713,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) } } } else { - requestWhois(tPtr,destination); + requestWhois(tPtr,now,destination); return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly } diff --git a/node/Switch.hpp b/node/Switch.hpp index 88415541..2420607d 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -111,9 +111,10 @@ public: * Request WHOIS on a given address * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call + * @param now Current time * @param addr Address to look up */ - void requestWhois(void *tPtr,const Address &addr); + void requestWhois(void *tPtr,const uint64_t now,const Address &addr); /** * Run any processes that are waiting for this peer's identity @@ -139,34 +140,27 @@ public: private: bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination); - Address _sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted); bool _trySend(void *tPtr,Packet &packet,bool encrypt); // packet is modified if return is true const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; + uint64_t _lastCheckedQueues; - // Outstanding WHOIS requests and how many retries they've undergone - struct WhoisRequest - { - WhoisRequest() : lastSent(0),retries(0) {} - uint64_t lastSent; - Address peersConsulted[ZT_MAX_WHOIS_RETRIES]; // by retry - unsigned int retries; // 0..ZT_MAX_WHOIS_RETRIES - }; - Hashtable< Address,WhoisRequest > _outstandingWhoisRequests; - Mutex _outstandingWhoisRequests_m; + // Time we last sent a WHOIS request for each address + Hashtable< Address,uint64_t > _lastSentWhoisRequest; + Mutex _lastSentWhoisRequest_m; // Packets waiting for WHOIS replies or other decode info or missing fragments struct RXQueueEntry { RXQueueEntry() : timestamp(0) {} - uint64_t timestamp; // 0 if entry is not in use - uint64_t packetId; + volatile uint64_t timestamp; // 0 if entry is not in use + volatile uint64_t packetId; IncomingPacket frag0; // head of packet Packet::Fragment frags[ZT_MAX_PACKET_FRAGMENTS - 1]; // later fragments (if any) unsigned int totalFragments; // 0 if only frag0 received, waiting for frags uint32_t haveFragments; // bit mask, LSB to MSB - bool complete; // if true, packet is complete + volatile bool complete; // if true, packet is complete }; RXQueueEntry _rxQueue[ZT_RX_QUEUE_SIZE]; AtomicCounter _rxQueuePtr; diff --git a/node/Tag.cpp b/node/Tag.cpp index 39b17f2a..bde41a70 100644 --- a/node/Tag.cpp +++ b/node/Tag.cpp @@ -30,6 +30,7 @@ #include "Topology.hpp" #include "Switch.hpp" #include "Network.hpp" +#include "Node.hpp" namespace ZeroTier { @@ -39,7 +40,7 @@ int Tag::verify(const RuntimeEnvironment *RR,void *tPtr) const return -1; const Identity id(RR->topology->getIdentity(tPtr,_signedBy)); if (!id) { - RR->sw->requestWhois(tPtr,_signedBy); + RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy); return 1; } try { diff --git a/node/Topology.cpp b/node/Topology.cpp index aeca59a7..ee5d969d 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -154,13 +154,11 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta) return Identity(); } -SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid) +SharedPtr Topology::getUpstreamPeer() { const uint64_t now = RR->node->now(); - unsigned int bestQualityOverall = ~((unsigned int)0); - unsigned int bestQualityNotAvoid = ~((unsigned int)0); - const SharedPtr *bestOverall = (const SharedPtr *)0; - const SharedPtr *bestNotAvoid = (const SharedPtr *)0; + unsigned int bestq = ~((unsigned int)0); + const SharedPtr *best = (const SharedPtr *)0; Mutex::Lock _l1(_peers_m); Mutex::Lock _l2(_upstreams_m); @@ -168,32 +166,17 @@ SharedPtr Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi for(std::vector
::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { const SharedPtr *p = _peers.get(*a); if (p) { - bool avoiding = false; - for(unsigned int i=0;iaddress()) { - avoiding = true; - break; - } - } const unsigned int q = (*p)->relayQuality(now); - if (q <= bestQualityOverall) { - bestQualityOverall = q; - bestOverall = &(*p); - } - if ((!avoiding)&&(q <= bestQualityNotAvoid)) { - bestQualityNotAvoid = q; - bestNotAvoid = &(*p); + if (q <= bestq) { + bestq = q; + best = p; } } } - if (bestNotAvoid) { - return *bestNotAvoid; - } else if ((!strictAvoid)&&(bestOverall)) { - return *bestOverall; - } - - return SharedPtr(); + if (!best) + return SharedPtr(); + return *best; } bool Topology::isUpstream(const Identity &id) const diff --git a/node/Topology.hpp b/node/Topology.hpp index 04dfb1cc..43921896 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -127,19 +127,9 @@ public: /** * Get the current best upstream peer * - * @return Root server with lowest latency or NULL if none + * @return Upstream or NULL if none available */ - inline SharedPtr getUpstreamPeer() { return getUpstreamPeer((const Address *)0,0,false); } - - /** - * Get the current best upstream peer, avoiding those in the supplied avoid list - * - * @param avoid Nodes to avoid - * @param avoidCount Number of nodes to avoid - * @param strictAvoid If false, consider avoided root servers anyway if no non-avoid root servers are available - * @return Root server or NULL if none available - */ - SharedPtr getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid); + SharedPtr getUpstreamPeer(); /** * @param id Identity to check -- cgit v1.2.3 From 180049a27725523e18004769d145ba88a68c799b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 16:55:22 -0700 Subject: Fix pointer bug. --- node/Switch.cpp | 3 +-- node/Switch.hpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 8446602c..388ed672 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -620,8 +620,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) _txQueue.erase(txi++); } else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { RR->t->txTimedOut(tPtr,txi->dest); - _txQueue.erase(txi); - ++txi; + _txQueue.erase(txi++); } else if (!RR->topology->getPeer(tPtr,txi->dest)) { requestWhois(tPtr,now,txi->dest); ++txi; diff --git a/node/Switch.hpp b/node/Switch.hpp index 2420607d..c258a255 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -144,7 +144,7 @@ private: const RuntimeEnvironment *const RR; uint64_t _lastBeaconResponse; - uint64_t _lastCheckedQueues; + volatile uint64_t _lastCheckedQueues; // Time we last sent a WHOIS request for each address Hashtable< Address,uint64_t > _lastSentWhoisRequest; -- cgit v1.2.3 From 0a9c3b557181a77f059d4b2957f6a02f7cc5bde5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 16:59:31 -0700 Subject: Fix possible deadlock. --- node/Switch.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 388ed672..62c3f450 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -613,6 +613,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) return (unsigned long)(ZT_WHOIS_RETRY_DELAY - timeSinceLastCheck); _lastCheckedQueues = now; + std::vector
needWhois; { Mutex::Lock _l(_txQueue_m); for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) { @@ -621,14 +622,15 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) } else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) { RR->t->txTimedOut(tPtr,txi->dest); _txQueue.erase(txi++); - } else if (!RR->topology->getPeer(tPtr,txi->dest)) { - requestWhois(tPtr,now,txi->dest); - ++txi; } else { + if (!RR->topology->getPeer(tPtr,txi->dest)) + needWhois.push_back(txi->dest); ++txi; } } } + for(std::vector
::const_iterator i(needWhois.begin());i!=needWhois.end();++i) + requestWhois(tPtr,now,*i); for(unsigned int ptr=0;ptr Date: Wed, 23 Aug 2017 17:14:06 -0700 Subject: Fix another deadlock. --- node/Switch.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'node') diff --git a/node/Switch.cpp b/node/Switch.cpp index 62c3f450..fce12622 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -544,11 +544,16 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const void Switch::send(void *tPtr,Packet &packet,bool encrypt) { - if (packet.destination() == RR->identity.address()) + const Address dest(packet.destination()); + if (dest == RR->identity.address()) return; if (!_trySend(tPtr,packet,encrypt)) { - Mutex::Lock _l(_txQueue_m); - _txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt)); + { + Mutex::Lock _l(_txQueue_m); + _txQueue.push_back(TXQueueEntry(dest,RR->node->now(),packet,encrypt)); + } + if (!RR->topology->getPeer(tPtr,dest)) + requestWhois(tPtr,RR->node->now(),dest); } } @@ -714,7 +719,6 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) } } } else { - requestWhois(tPtr,now,destination); return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly } -- cgit v1.2.3 From dd8b03a5c55baf7349ce075c8f4cad1d59cbe988 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 18:28:40 -0700 Subject: Threading issue fix? --- node/Peer.cpp | 7 +++++-- node/Switch.cpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index 127f222c..b3020854 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -424,18 +424,21 @@ void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remo SharedPtr op; SharedPtr np(RR->topology->getPath(localSocket,remoteAddress)); + np->received(now); attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,np->nextOutgoingCounter()); { Mutex::Lock _l(_paths_m); if (remoteAddress.ss_family == AF_INET) { op = _v4Path.p; - _v4Path.p = np; + _v4Path.lr = now; _v4Path.sticky = now; + _v4Path.p = np; } else if (remoteAddress.ss_family == AF_INET6) { op = _v6Path.p; - _v6Path.p = np; + _v6Path.lr = now; _v6Path.sticky = now; + _v6Path.p = np; } } diff --git a/node/Switch.cpp b/node/Switch.cpp index fce12622..952bdef8 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -719,7 +719,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) } } } else { - return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly + return false; } unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); -- cgit v1.2.3 From 49fa30d495c0524a15ba86668f4fee12b9715089 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 23 Aug 2017 18:52:32 -0700 Subject: Ticket lock for x64/gcc/clang platforms. --- node/Mutex.hpp | 84 +++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 21 deletions(-) (limited to 'node') diff --git a/node/Mutex.hpp b/node/Mutex.hpp index 854f321a..b16f53f7 100644 --- a/node/Mutex.hpp +++ b/node/Mutex.hpp @@ -32,32 +32,84 @@ #ifdef __UNIX_LIKE__ +#include #include #include namespace ZeroTier { +#if defined(__GNUC__) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64)) + +// Inline ticket lock on x64 systems with GCC and CLANG (Mac, Linux) -- this is really fast as long as locking durations are very short class Mutex : NonCopyable { public: - Mutex() + Mutex() : + nextTicket(0), + nowServing(0) { - pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0); } - ~Mutex() + inline void lock() const { - pthread_mutex_destroy(&_mh); + const uint16_t myTicket = __sync_fetch_and_add(&(const_cast(this)->nextTicket),1); + while (nowServing != myTicket) { + __asm__ __volatile__("rep;nop"::); + __asm__ __volatile__("":::"memory"); + } } - inline void lock() + inline void unlock() const { - pthread_mutex_lock(&_mh); + ++(const_cast(this)->nowServing); } - inline void unlock() + /** + * Uses C++ contexts and constructor/destructor to lock/unlock automatically + */ + class Lock : NonCopyable { - pthread_mutex_unlock(&_mh); + public: + Lock(Mutex &m) : + _m(&m) + { + m.lock(); + } + + Lock(const Mutex &m) : + _m(const_cast(&m)) + { + _m->lock(); + } + + ~Lock() + { + _m->unlock(); + } + + private: + Mutex *const _m; + }; + +private: + uint16_t nextTicket; + uint16_t nowServing; +}; + +#else + +// libpthread based mutex lock +class Mutex : NonCopyable +{ +public: + Mutex() + { + pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0); + } + + ~Mutex() + { + pthread_mutex_destroy(&_mh); } inline void lock() const @@ -70,9 +122,6 @@ public: (const_cast (this))->unlock(); } - /** - * Uses C++ contexts and constructor/destructor to lock/unlock automatically - */ class Lock : NonCopyable { public: @@ -101,6 +150,8 @@ private: pthread_mutex_t _mh; }; +#endif + } // namespace ZeroTier #endif // Apple / Linux @@ -112,6 +163,7 @@ private: namespace ZeroTier { +// Windows critical section based lock class Mutex : NonCopyable { public: @@ -125,16 +177,6 @@ public: DeleteCriticalSection(&_cs); } - inline void lock() - { - EnterCriticalSection(&_cs); - } - - inline void unlock() - { - LeaveCriticalSection(&_cs); - } - inline void lock() const { (const_cast (this))->lock(); -- cgit v1.2.3 From 02ed84774c2b9eb23995d722349476a1e5a6870f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 31 Aug 2017 20:47:44 -0400 Subject: Non-x86 build fix. --- node/Mutex.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/Mutex.hpp b/node/Mutex.hpp index b16f53f7..b68e5d93 100644 --- a/node/Mutex.hpp +++ b/node/Mutex.hpp @@ -114,12 +114,12 @@ public: inline void lock() const { - (const_cast (this))->lock(); + pthread_mutex_lock(&((const_cast (this))->_mh)); } inline void unlock() const { - (const_cast (this))->unlock(); + pthread_mutex_unlock(&((const_cast (this))->_mh)); } class Lock : NonCopyable -- cgit v1.2.3 From b1fb020aea80ea70ed801b876ad01beb09e218e1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 1 Sep 2017 10:43:44 -0700 Subject: Raise chunk size to max packet size for network configs. Chunking breaks really ancient clients, so this helps them live a little longer. No real downside for new clients. --- node/Node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Node.cpp b/node/Node.cpp index 09260172..34609fd4 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -592,7 +592,7 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de const unsigned int totalSize = dconf->sizeBytes(); unsigned int chunkIndex = 0; while (chunkIndex < totalSize) { - const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256))); + const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PACKET_IDX_PAYLOAD + 256))); Packet outp(destination,RR->identity.address(),(requestPacketId) ? Packet::VERB_OK : Packet::VERB_NETWORK_CONFIG); if (requestPacketId) { outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST); -- cgit v1.2.3 From 2d858b05ac8554ba11374fefaeb583a0bbc0546b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 1 Sep 2017 12:03:31 -0700 Subject: Another fix for ye old tyme clients. --- include/ZeroTierOne.h | 11 ++++++++++- node/Constants.hpp | 9 --------- node/NetworkConfig.cpp | 5 +++-- 3 files changed, 13 insertions(+), 12 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 2a970d5b..b889ade0 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -92,6 +92,15 @@ extern "C" { */ #define ZT_MAX_MTU 10000 +/** + * Default payload MTU for UDP packets + * + * This is 1500 - IPv6 UDP overhead - PPPoE overhead and is safe for 99.9% of + * all Internet links. + */ +#define ZT_DEFAULT_PHYSMTU 1444 +#define ZT_UDP_DEFAULT_PAYLOAD_MTU 1444 + /** * Maximum physical UDP payload */ @@ -103,7 +112,7 @@ extern "C" { #define ZT_MAX_HEADROOM 224 /** - * Maximum physical MTU + * Maximum payload MTU for UDP packets */ #define ZT_MAX_PHYSMTU (ZT_MAX_PHYSPAYLOAD + ZT_MAX_HEADROOM) diff --git a/node/Constants.hpp b/node/Constants.hpp index cda1af3b..651fe50d 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -171,15 +171,6 @@ */ #define ZT_ADDRESS_RESERVED_PREFIX 0xff -/** - * Default payload MTU for UDP packets - * - * In the future we might support UDP path MTU discovery, but for now we - * set a maximum that is equal to 1500 minus 8 (for PPPoE overhead, common - * in some markets) minus 48 (IPv6 UDP overhead). - */ -#define ZT_UDP_DEFAULT_PAYLOAD_MTU 1444 - /** * Default MTU used for Ethernet tap device */ diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp index 0bf4bc19..110a20b0 100644 --- a/node/NetworkConfig.cpp +++ b/node/NetworkConfig.cpp @@ -35,6 +35,7 @@ namespace ZeroTier { bool NetworkConfig::toDictionary(Dictionary &d,bool includeLegacy) const { Buffer *tmp = new Buffer(); + char tmp2[128]; try { d.clear(); @@ -46,8 +47,8 @@ bool NetworkConfig::toDictionary(Dictionary &d,b if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false; - if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget)) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo.toString(tmp2))) return false; + if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget.toString(tmp2))) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false; if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false; -- cgit v1.2.3 From f8014413a376551b7853baae81072f969a755e46 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 1 Sep 2017 16:25:34 -0700 Subject: Add UDP MTU configurability. --- include/ZeroTierOne.h | 55 +++++++++++++++-------------- node/Multicaster.cpp | 2 +- node/Network.cpp | 1 - node/Node.cpp | 14 ++++---- node/Node.hpp | 2 +- node/Packet.hpp | 6 +--- node/Switch.cpp | 13 ++++--- node/Topology.cpp | 2 +- node/Topology.hpp | 95 ++++++++++++++++++++++++++++++++++++++------------ service/OneService.cpp | 25 ++++++------- service/README.md | 3 +- 11 files changed, 134 insertions(+), 84 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index b889ade0..7cbebb32 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -93,13 +93,17 @@ extern "C" { #define ZT_MAX_MTU 10000 /** - * Default payload MTU for UDP packets + * Minimum UDP payload size allowed + */ +#define ZT_MIN_PHYSMTU 1400 + +/** + * Default UDP payload size (physical path MTU) not including UDP and IP overhead * * This is 1500 - IPv6 UDP overhead - PPPoE overhead and is safe for 99.9% of * all Internet links. */ #define ZT_DEFAULT_PHYSMTU 1444 -#define ZT_UDP_DEFAULT_PAYLOAD_MTU 1444 /** * Maximum physical UDP payload @@ -172,9 +176,9 @@ extern "C" { #define ZT_MAX_PEER_NETWORK_PATHS 4 /** - * Maximum number of trusted physical network paths + * Maximum number of path configurations that can be set */ -#define ZT_MAX_TRUSTED_PATHS 16 +#define ZT_MAX_CONFIGURABLE_PATHS 32 /** * Maximum number of rules per capability @@ -1058,11 +1062,6 @@ typedef struct */ unsigned int mtu; - /** - * Recommended MTU to avoid fragmentation at the physical layer (hint) - */ - unsigned int physicalMtu; - /** * If nonzero, the network this port belongs to indicates DHCP availability * @@ -1132,6 +1131,21 @@ typedef struct unsigned long networkCount; } ZT_VirtualNetworkList; +/** + * Physical path configuration + */ +typedef struct { + /** + * If non-zero set this physical network path to be trusted to disable encryption and authentication + */ + uint64_t trustedPathId; + + /** + * Physical path MTU from ZT_MIN_PHYSMTU and ZT_MAX_PHYSMTU or <= 0 to use default + */ + int mtu; +} ZT_PhysicalPathConfiguration; + /** * Physical network path to a peer */ @@ -1856,27 +1870,14 @@ ZT_SDK_API int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,ui ZT_SDK_API void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); /** - * Set trusted paths - * - * A trusted path is a physical network (network/bits) over which both - * encryption and authentication can be skipped to improve performance. - * Each trusted path must have a non-zero unique ID that is the same across - * all participating nodes. - * - * We don't recommend using trusted paths at all unless you really *need* - * near-bare-metal performance. Even on a LAN authentication and encryption - * are never a bad thing, and anything that introduces an "escape hatch" - * for encryption should be treated with the utmost care. - * - * Calling with NULL pointers for networks and ids and a count of zero clears - * all trusted paths. + * Set configuration for a given physical path * * @param node Node instance - * @param networks Array of [count] networks - * @param ids Array of [count] corresponding non-zero path IDs (zero path IDs are ignored) - * @param count Number of trusted paths-- values greater than ZT_MAX_TRUSTED_PATHS are clipped + * @param pathNetwork Network/CIDR of path or NULL to clear the cache and reset all paths to default + * @param pathConfig Path configuration or NULL to erase this entry and therefore reset it to NULL + * @return OK or error code */ -ZT_SDK_API void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); +ZT_SDK_API enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig); /** * Get ZeroTier One version diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index fb7b068f..e8c8613a 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -111,7 +111,7 @@ unsigned int Multicaster::gather(const Address &queryingPeer,uint64_t nwid,const // Members are returned in random order so that repeated gather queries // will return different subsets of a large multicast group. k = 0; - while ((added < limit)&&(k < s->members.size())&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_UDP_DEFAULT_PAYLOAD_MTU)) { + while ((added < limit)&&(k < s->members.size())&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_PROTO_MAX_PACKET_LENGTH)) { rptr = (unsigned int)RR->node->prng(); restart_member_scan: diff --git a/node/Network.cpp b/node/Network.cpp index f7b144e3..16155c33 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -1346,7 +1346,6 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const ec->status = _status(); ec->type = (_config) ? (_config.isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE; ec->mtu = (_config) ? _config.mtu : ZT_DEFAULT_MTU; - ec->physicalMtu = ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 16); ec->dhcp = 0; std::vector
ab(_config.activeBridges()); ec->bridge = ((_config.allowPassiveBridging())||(std::find(ab.begin(),ab.end(),RR->identity.address()) != ab.end())) ? 1 : 0; diff --git a/node/Node.cpp b/node/Node.cpp index 34609fd4..871fb21b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -561,9 +561,9 @@ uint64_t Node::prng() return z + y; } -void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count) +ZT_ResultCode Node::setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork, const ZT_PhysicalPathConfiguration *pathConfig) { - RR->topology->setTrustedPaths(reinterpret_cast(networks),ids,count); + return ZT_RESULT_OK; } World Node::planet() const @@ -815,7 +815,7 @@ enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,u } } -ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId) +enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId) { try { return reinterpret_cast(node)->deorbit(tptr,moonWorldId); @@ -902,11 +902,13 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count) +enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig) { try { - reinterpret_cast(node)->setTrustedPaths(networks,ids,count); - } catch ( ... ) {} + return reinterpret_cast(node)->setPhysicalPathConfiguration(pathNetwork,pathConfig); + } catch ( ... ) { + return ZT_RESULT_FATAL_ERROR_INTERNAL; + } } void ZT_version(int *major,int *minor,int *revision) diff --git a/node/Node.hpp b/node/Node.hpp index 9658174f..1aa01c9a 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -192,7 +192,7 @@ public: inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast(&addr)) != 0) : false ); } uint64_t prng(); - void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count); + ZT_ResultCode setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig); World planet() const; std::vector moons() const; diff --git a/node/Packet.hpp b/node/Packet.hpp index 5fc631b1..db70e06f 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -225,12 +225,8 @@ /** * Packet buffer size (can be changed) - * - * The current value is big enough for ZT_MAX_PACKET_FRAGMENTS, the pragmatic - * packet fragment limit, times the default UDP MTU. Most packets won't be - * this big. */ -#define ZT_PROTO_MAX_PACKET_LENGTH (ZT_MAX_PACKET_FRAGMENTS * ZT_UDP_DEFAULT_PAYLOAD_MTU) +#define ZT_PROTO_MAX_PACKET_LENGTH (ZT_MAX_PACKET_FRAGMENTS * ZT_DEFAULT_PHYSMTU) /** * Minimum viable packet length (a.k.a. header length) diff --git a/node/Switch.cpp b/node/Switch.cpp index 952bdef8..f46b3e73 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -722,10 +722,13 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) return false; } - unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU); + unsigned int mtu = ZT_DEFAULT_PHYSMTU; + uint64_t trustedPathId = 0; + RR->topology->getOutboundPathInfo(viaPath->address(),mtu,trustedPathId); + + unsigned int chunkSize = std::min(packet.size(),mtu); packet.setFragmented(chunkSize < packet.size()); - const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address()); if (trustedPathId) { packet.setTrusted(trustedPathId); } else { @@ -737,13 +740,13 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) // Too big for one packet, fragment the rest unsigned int fragStart = chunkSize; unsigned int remaining = packet.size() - chunkSize; - unsigned int fragsRemaining = (remaining / (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)); - if ((fragsRemaining * (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining) + unsigned int fragsRemaining = (remaining / (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH)); + if ((fragsRemaining * (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining) ++fragsRemaining; const unsigned int totalFragments = fragsRemaining + 1; for(unsigned int fno=1;fnosend(RR,tPtr,frag.data(),frag.size(),now); fragStart += chunkSize; diff --git a/node/Topology.cpp b/node/Topology.cpp index ee5d969d..905b6a91 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -65,7 +65,7 @@ static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x0 Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) : RR(renv), - _trustedPathCount(0), + _numConfiguredPhysicalPaths(0), _amRoot(false) { uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH]; diff --git a/node/Topology.hpp b/node/Topology.hpp index 43921896..34df28a1 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -339,6 +339,41 @@ public: */ inline bool amRoot() const { return _amRoot; } + /** + * Get info about a path + * + * The supplied result variables are not modified if no special config info is found. + * + * @param physicalAddress Physical endpoint address + * @param mtu Variable set to MTU + * @param trustedPathId Variable set to trusted path ID + */ + inline void getOutboundPathInfo(const InetAddress &physicalAddress,unsigned int &mtu,uint64_t &trustedPathId) + { + for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i ZT_MAX_TRUSTED_PATHS) - count = ZT_MAX_TRUSTED_PATHS; - Mutex::Lock _l(_trustedPaths_m); - for(unsigned int i=0;i cpaths; + for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i ZT_MAX_PHYSMTU) + pc.mtu = ZT_MAX_PHYSMTU; + + cpaths[*(reinterpret_cast(pathNetwork))] = pc; + } else { + cpaths.erase(*(reinterpret_cast(pathNetwork))); + } + + unsigned int cnt = 0; + for(std::map::const_iterator i(cpaths.begin());((i!=cpaths.end())&&(cntfirst; + _physicalPathConfig[cnt].second = i->second; + ++cnt; + } + _numConfiguredPhysicalPaths = cnt; } - _trustedPathCount = count; } /** @@ -414,10 +467,8 @@ private: const RuntimeEnvironment *const RR; - uint64_t _trustedPathIds[ZT_MAX_TRUSTED_PATHS]; - InetAddress _trustedPathNetworks[ZT_MAX_TRUSTED_PATHS]; - unsigned int _trustedPathCount; - Mutex _trustedPaths_m; + std::pair _physicalPathConfig[ZT_MAX_CONFIGURABLE_PATHS]; + volatile unsigned int _numConfiguredPhysicalPaths; Hashtable< Address,SharedPtr > _peers; Mutex _peers_m; diff --git a/service/OneService.cpp b/service/OneService.cpp index 093d9ee8..66e9a9c8 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -564,16 +564,14 @@ public: // Read local configuration { - uint64_t trustedPathIds[ZT_MAX_TRUSTED_PATHS]; - InetAddress trustedPathNetworks[ZT_MAX_TRUSTED_PATHS]; - unsigned int trustedPathCount = 0; + std::map ppc; // LEGACY: support old "trustedpaths" flat file FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r"); if (trustpaths) { fprintf(stderr,"WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S); char buf[1024]; - while ((fgets(buf,sizeof(buf),trustpaths))&&(trustedPathCount < ZT_MAX_TRUSTED_PATHS)) { + while (fgets(buf,sizeof(buf),trustpaths)) { int fno = 0; char *saveptr = (char *)0; uint64_t trustedPathId = 0; @@ -587,9 +585,8 @@ public: ++fno; } if ( (trustedPathId != 0) && ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathNetwork.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (trustedPathNetwork.netmaskBits() > 0) ) { - trustedPathIds[trustedPathCount] = trustedPathId; - trustedPathNetworks[trustedPathCount] = trustedPathNetwork; - ++trustedPathCount; + ppc[trustedPathNetwork].trustedPathId = trustedPathId; + ppc[trustedPathNetwork].mtu = 0; // use default } } fclose(trustpaths); @@ -618,12 +615,10 @@ public: if (phy.value().is_object()) { uint64_t tpid; if ((tpid = OSUtils::jsonInt(phy.value()["trustedPathId"],0ULL)) != 0ULL) { - if ( ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (net.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (net.netmaskBits() > 0) ) { - trustedPathIds[trustedPathCount] = tpid; - trustedPathNetworks[trustedPathCount] = net; - ++trustedPathCount; - } + if ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) + ppc[net].trustedPathId = tpid; } + ppc[net].mtu = (int)OSUtils::jsonInt(phy.value()["mtu"],0ULL); // 0 means use default } } } @@ -638,8 +633,10 @@ public: } // Set trusted paths if there are any - if (trustedPathCount) - _node->setTrustedPaths(reinterpret_cast(trustedPathNetworks),trustedPathIds,trustedPathCount); + if (ppc.size() > 0) { + for(std::map::iterator i(ppc.begin());i!=ppc.end();++i) + _node->setPhysicalPathConfiguration(reinterpret_cast(&(i->first)),&(i->second)); + } } // Apply other runtime configuration from local.conf diff --git a/service/README.md b/service/README.md index f5223f2d..77085a6f 100644 --- a/service/README.md +++ b/service/README.md @@ -14,7 +14,8 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al "physical": { /* Settings that apply to physical L2/L3 network paths. */ "NETWORK/bits": { /* Network e.g. 10.0.0.0/24 or fd00::/32 */ "blacklist": true|false, /* If true, blacklist this path for all ZeroTier traffic */ - "trustedPathId": 0|!0 /* If present and nonzero, define this as a trusted path (see below) */ + "trustedPathId": 0|!0, /* If present and nonzero, define this as a trusted path (see below) */ + "mtu": 0|!0 /* if present and non-zero, set UDP maximum payload MTU for this path */ } /* ,... additional networks */ }, "virtual": { /* Settings applied to ZeroTier virtual network devices (VL1) */ -- cgit v1.2.3 From 52916eebcfae2559966d12d4be4b5376289a982d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 14 Sep 2017 20:56:50 -0700 Subject: Keep attemting to upgrade direct path if path is not private to facilitate better use of LANs and backplane networks. --- node/Peer.cpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'node') diff --git a/node/Peer.cpp b/node/Peer.cpp index b3020854..a954b716 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -194,8 +194,12 @@ void Peer::received( } } } - } else if (this->trustEstablished(now)) { - // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) + } + + // If we are being relayed or if we're using a global address, send PUSH_DIRECT_PATHS. + // In the global address case we push only configured direct paths to accomplish + // fall-forward to local backplane networks over e.g. LAN or Amazon VPC. + if ( ((hops > 0)||(path->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) && (this->trustEstablished(now)) ) { if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) { _lastDirectPathPushSent = now; @@ -205,13 +209,15 @@ void Peer::received( for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) pathsToPush.push_back(*i); - std::vector sym(RR->sa->getSymmetricNatPredictions()); - for(unsigned long i=0,added=0;inode->prng() % sym.size()]); - if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { - pathsToPush.push_back(tmp); - if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) - break; + if (hops > 0) { + std::vector sym(RR->sa->getSymmetricNatPredictions()); + for(unsigned long i=0,added=0;inode->prng() % sym.size()]); + if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) { + pathsToPush.push_back(tmp); + if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) + break; + } } } -- cgit v1.2.3 From 302c15140e90df438988bb648c4c71a5b910fc87 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 25 Sep 2017 08:53:47 -0700 Subject: Fix API problem with path configuration.. --- include/ZeroTierOne.h | 15 +++++++++++++++ node/Node.cpp | 1 + 2 files changed, 16 insertions(+) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 7cbebb32..cf6b21fd 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -803,6 +803,21 @@ typedef struct uint8_t mask; } ipv4; + /** + * Integer range match in packet payload + * + * This allows matching of ranges of integers up to 64 bits wide where + * the range is +/- INT32_MAX. It's packed this way so it fits in 16 + * bytes and doesn't enlarge the overall size of this union. + */ + struct { + uint64_t start; // integer range start + int32_t delta; // +/- offset from start of integer range end + uint16_t idx; // index in packet of integer + uint8_t bits; // number of bits in integer (range: 1-64) + uint8_t endian; // endianness of integer in packet (0 == big, 1 == little) + } intRange; + /** * Packet characteristic flags being matched */ diff --git a/node/Node.cpp b/node/Node.cpp index 871fb21b..cc076e4d 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -563,6 +563,7 @@ uint64_t Node::prng() ZT_ResultCode Node::setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork, const ZT_PhysicalPathConfiguration *pathConfig) { + RR->topology->setPhysicalPathConfiguration(pathNetwork,pathConfig); return ZT_RESULT_OK; } -- cgit v1.2.3 From 9c903567bb1ae5038f57f198a4db4db21aedbd87 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 25 Sep 2017 13:42:19 -0700 Subject: Hashtable needs to include Constants.hpp for ZT_EXCEPTION_OUT_OF_MEMORY --- node/Hashtable.hpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'node') diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index eff5b62a..95a8e74f 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -27,6 +27,8 @@ #ifndef ZT_HASHTABLE_HPP #define ZT_HASHTABLE_HPP +#include "Constants.hpp" + #include #include #include -- cgit v1.2.3 From 239c2540d6798b897944c7a719f27d61479e21cc Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Wed, 27 Sep 2017 15:05:13 -0700 Subject: Mutex::lock and ::unlock just called themselves --- node/Mutex.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'node') diff --git a/node/Mutex.hpp b/node/Mutex.hpp index b68e5d93..53ae05c4 100644 --- a/node/Mutex.hpp +++ b/node/Mutex.hpp @@ -177,6 +177,16 @@ public: DeleteCriticalSection(&_cs); } + inline void lock() + { + EnterCriticalSection(&_cs); + } + + inline void unlock() + { + LeaveCriticalSection(&_cs); + } + inline void lock() const { (const_cast (this))->lock(); -- cgit v1.2.3 From e564c56dce6a6ad4dccb68af2c6c8d4a7148a298 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 28 Sep 2017 10:39:43 -0700 Subject: Set size of buffer after setting data with unsafeData() call --- node/Topology.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index 905b6a91..bfa62ff5 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -127,6 +127,7 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) uint64_t idbuf[2]; idbuf[0] = zta.toInt(); idbuf[1] = 0; int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER,idbuf,buf.unsafeData(),ZT_PEER_MAX_SERIALIZED_STATE_SIZE); if (len > 0) { + buf.setSize(len); Mutex::Lock _l(_peers_m); SharedPtr &ap = _peers[zta]; if (ap) -- cgit v1.2.3 From 7cf70d111ac3bfc1bf6aa7e3ec8d9c83e35f48a1 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 28 Sep 2017 10:40:27 -0700 Subject: Return an explicit NULL shared pointer at the end of Topology::getPeer() --- node/Topology.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'node') diff --git a/node/Topology.cpp b/node/Topology.cpp index bfa62ff5..8a830b93 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -135,7 +135,7 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) ap = Peer::deserializeFromCache(RR->node->now(),tPtr,buf,RR); if (!ap) _peers.erase(zta); - return ap; + return SharedPtr(); } } catch ( ... ) {} // ignore invalid identities or other strage failures -- cgit v1.2.3 From b1d60df44cb24589bc5718da932ef4bb54168fa3 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 2 Oct 2017 15:52:57 -0700 Subject: timestamps changed from uint64_t to int64_t There were cases in the code where time calculations and comparisons were overflowing and causing connection instability. This will keep time calculations within expected ranges. --- controller/EmbeddedNetworkController.cpp | 14 +++--- controller/EmbeddedNetworkController.hpp | 8 ++-- controller/JSONDB.cpp | 4 +- controller/JSONDB.hpp | 2 +- include/ZeroTierOne.h | 12 ++--- node/CertificateOfMembership.hpp | 2 +- node/IncomingPacket.cpp | 12 ++--- node/IncomingPacket.hpp | 4 +- node/Membership.cpp | 8 ++-- node/Membership.hpp | 14 +++--- node/Multicaster.cpp | 8 ++-- node/Multicaster.hpp | 12 ++--- node/Network.cpp | 10 ++--- node/Network.hpp | 4 +- node/Node.cpp | 34 +++++++------- node/Node.hpp | 26 +++++------ node/OutboundMulticast.hpp | 2 +- node/Path.cpp | 2 +- node/Path.hpp | 22 ++++----- node/Peer.cpp | 35 ++++++++------- node/Peer.hpp | 76 ++++++++++++++++---------------- node/Revocation.hpp | 4 +- node/SelfAwareness.cpp | 6 +-- node/SelfAwareness.hpp | 4 +- node/Switch.cpp | 20 ++++----- node/Switch.hpp | 14 +++--- node/Topology.cpp | 7 +-- node/Topology.hpp | 4 +- one.cpp | 6 +-- osdep/OSUtils.hpp | 6 +-- osdep/PortMapper.cpp | 2 +- service/OneService.cpp | 22 +++++---- service/SoftwareUpdater.cpp | 2 +- service/SoftwareUpdater.hpp | 2 +- 34 files changed, 209 insertions(+), 201 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 1d46d5e6..20f81966 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -535,7 +535,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } else { // Get network - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); JSONDB::NetworkSummaryInfo ns; _db.getNetworkSummaryInfo(nwid,ns); _addNetworkNonPersistedFields(network,now,ns); @@ -602,7 +602,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( responseContentType = "application/json"; return 400; } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); if (path[0] == "network") { @@ -1087,7 +1087,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) } } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.10llx-%.16llx-%.8lx",_signingId.address().toInt(),rt.origin,now,++idCounter); d["id"] = id; d["objtype"] = "trace"; @@ -1129,7 +1129,7 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt) void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId) { // Send an update to all members of the network that are online - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); Mutex::Lock _l(_memberStatus_m); for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { if ((i->first.networkId == networkId)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) @@ -1150,7 +1150,7 @@ void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,c void EmbeddedNetworkController::onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId) { - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); Revocation rev((uint32_t)_node->prng(),networkId,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(memberId),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); { @@ -1224,7 +1224,7 @@ void EmbeddedNetworkController::_request( if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); if (requestPacketId) { Mutex::Lock _l(_memberStatus_m); @@ -1360,7 +1360,7 @@ void EmbeddedNetworkController::_request( // If we made it this far, they are authorized. // ------------------------------------------------------------------------- - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; + int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; if (now > ns.mostRecentDeauthTime) { // If we recently de-authorized a member, shrink credential TTL/max delta to // be below the threshold required to exclude it. Cap this to a min/max to diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 19469164..fce56065 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -166,7 +166,7 @@ private: } network["objtype"] = "network"; } - inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const JSONDB::NetworkSummaryInfo &ns) + inline void _addNetworkNonPersistedFields(nlohmann::json &network,int64_t now,const JSONDB::NetworkSummaryInfo &ns) { network["clock"] = now; network["authorizedMemberCount"] = ns.authorizedMemberCount; @@ -182,7 +182,7 @@ private: // legacy fields network.erase("lastModified"); } - inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,uint64_t now) + inline void _addMemberNonPersistedFields(uint64_t nwid,uint64_t nodeId,nlohmann::json &member,int64_t now) { member["clock"] = now; Mutex::Lock _l(_memberStatus_m); @@ -197,7 +197,7 @@ private: member.erase("lastRequestMetaData"); } - const uint64_t _startTime; + const int64_t _startTime; volatile bool _running; BlockingQueue<_RQEntry *> _queue; @@ -230,7 +230,7 @@ private: Dictionary lastRequestMetaData; Identity identity; InetAddress physicalAddr; // last known physical address - inline bool online(const uint64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } + inline bool online(const int64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } }; struct _MemberStatusHash { diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index f362acf3..67b13393 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -323,7 +323,7 @@ void JSONDB::threadMain() _networks_m.unlock(); } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); try { Mutex::Lock _l(_networks_m); for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { @@ -373,7 +373,7 @@ void JSONDB::threadMain() } catch ( ... ) {} } else { try { - ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,OSUtils::jsonInt(member["lastDeauthorizedTime"],0ULL)); + ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,(int64_t)OSUtils::jsonInt(member["lastDeauthorizedTime"],0LL)); } catch ( ... ) {} } ++ns.totalMemberCount; diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 44f4d7f5..db909cb0 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -57,7 +57,7 @@ public: unsigned long authorizedMemberCount; unsigned long activeMemberCount; unsigned long totalMemberCount; - uint64_t mostRecentDeauthTime; + int64_t mostRecentDeauthTime; }; JSONDB(const std::string &basePath,EmbeddedNetworkController *parent); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index cf6b21fd..8adbc4d1 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1587,7 +1587,7 @@ struct ZT_Node_Callbacks * @param now Current clock in milliseconds * @return OK (0) or error code if a fatal error condition has occurred */ -ZT_SDK_API enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); +ZT_SDK_API enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now); /** * Delete a node and free all resources it consumes @@ -1615,12 +1615,12 @@ ZT_SDK_API void ZT_Node_delete(ZT_Node *node); ZT_SDK_API enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); /** * Process a frame from a virtual network port (tap) @@ -1641,7 +1641,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processWirePacket( ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -1649,7 +1649,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); /** * Perform periodic background operations @@ -1660,7 +1660,7 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( * @param nextBackgroundTaskDeadline Value/result: set to deadline for next call to processBackgroundTasks() * @return OK (0) or error code if a fatal error condition has occurred */ -ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); +ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline); /** * Join a network diff --git a/node/CertificateOfMembership.hpp b/node/CertificateOfMembership.hpp index 3ffa814f..0105fade 100644 --- a/node/CertificateOfMembership.hpp +++ b/node/CertificateOfMembership.hpp @@ -176,7 +176,7 @@ public: /** * @return Timestamp for this cert and maximum delta for timestamp */ - inline uint64_t timestamp() const + inline int64_t timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 685f2f09..c0409c91 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -169,7 +169,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar // Peers can send this in response to frames if they do not have a recent enough COM from us networkId = at(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD); const SharedPtr network(RR->node->network(networkId)); - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); if ( (network) && (network->config().com) && (peer->rateGateIncomingComRequest(now)) ) network->pushCredentialsNow(tPtr,peer->address(),now); } break; @@ -202,7 +202,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); const uint64_t pid = packetId(); const Address fromAddress(source()); @@ -210,7 +210,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool const unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION]; const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION]; const unsigned int vRevision = at(ZT_PROTO_VERB_HELLO_IDX_REVISION); - const uint64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); + const int64_t timestamp = at(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP); Identity id; unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY); @@ -725,7 +725,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); uint64_t authOnNetwork[256]; // cache for approved network IDs unsigned int authOnNetworkCount = 0; @@ -1082,7 +1082,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr, bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); // First, subject this to a rate limit if (!peer->rateGatePushDirectPaths(now)) { @@ -1186,7 +1186,7 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con void IncomingPacket::_sendErrorNeedCredentials(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer,const uint64_t nwid) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); if (peer->rateGateOutgoingComRequest(now)) { Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR); outp.append((uint8_t)verb()); diff --git a/node/IncomingPacket.hpp b/node/IncomingPacket.hpp index 45a0166d..c8f52721 100644 --- a/node/IncomingPacket.hpp +++ b/node/IncomingPacket.hpp @@ -77,7 +77,7 @@ public: * @param now Current time * @throws std::out_of_range Range error processing packet */ - IncomingPacket(const void *data,unsigned int len,const SharedPtr &path,uint64_t now) : + IncomingPacket(const void *data,unsigned int len,const SharedPtr &path,int64_t now) : Packet(data,len), _receiveTime(now), _path(path) @@ -93,7 +93,7 @@ public: * @param now Current time * @throws std::out_of_range Range error processing packet */ - inline void init(const void *data,unsigned int len,const SharedPtr &path,uint64_t now) + inline void init(const void *data,unsigned int len,const SharedPtr &path,int64_t now) { copyFrom(data,len); _receiveTime = now; diff --git a/node/Membership.cpp b/node/Membership.cpp index 17de6554..740f4e68 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -51,7 +51,7 @@ Membership::Membership() : resetPushState(); } -void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) +void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force) { bool sendCom = ( (nconf.com) && ( ((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) ); @@ -127,13 +127,13 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) { - const uint64_t newts = com.timestamp(); + const int64_t newts = com.timestamp(); if (newts <= _comRevocationThreshold) { RR->t->credentialRejected(tPtr,com,"revoked"); return ADD_REJECTED; } - const uint64_t oldts = _com.timestamp(); + const int64_t oldts = _com.timestamp(); if (newts < oldts) { RR->t->credentialRejected(tPtr,com,"old"); return ADD_REJECTED; @@ -227,7 +227,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -void Membership::clean(const uint64_t now,const NetworkConfig &nconf) +void Membership::clean(const int64_t now,const NetworkConfig &nconf) { _cleanCredImpl(nconf,_remoteTags); _cleanCredImpl(nconf,_remoteCaps); diff --git a/node/Membership.hpp b/node/Membership.hpp index c6e2b803..5612858a 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -80,7 +80,7 @@ public: * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none * @param force If true, send objects regardless of last push time */ - void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); + void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true @@ -88,7 +88,7 @@ public: * @param now Current time * @return True if we should update multicasts */ - inline bool multicastLikeGate(const uint64_t now) + inline bool multicastLikeGate(const int64_t now) { if ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD) { _lastUpdatedMulticast = now; @@ -110,7 +110,7 @@ public: return nconf.com.agreesWith(_com); } - inline bool recentlyAssociated(const uint64_t now) const + inline bool recentlyAssociated(const int64_t now) const { return ((_com)&&((now - _com.timestamp()) < ZT_PEER_ACTIVITY_TIMEOUT)); } @@ -180,7 +180,7 @@ public: * @param now Current time * @param nconf Current network configuration */ - void clean(const uint64_t now,const NetworkConfig &nconf); + void clean(const int64_t now,const NetworkConfig &nconf); /** * Reset last pushed time for local credentials @@ -223,13 +223,13 @@ private: } // Last time we pushed MULTICAST_LIKE(s) - uint64_t _lastUpdatedMulticast; + int64_t _lastUpdatedMulticast; // Last time we pushed our COM to this peer - uint64_t _lastPushedCom; + int64_t _lastPushedCom; // Revocation threshold for COM or 0 if none - uint64_t _comRevocationThreshold; + int64_t _comRevocationThreshold; // Remote member's latest network COM CertificateOfMembership _com; diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index e8c8613a..fa6f7bd1 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -51,7 +51,7 @@ Multicaster::~Multicaster() { } -void Multicaster::addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) +void Multicaster::addMultiple(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown) { const unsigned char *p = (const unsigned char *)addresses; const unsigned char *e = p + (5 * count); @@ -160,7 +160,7 @@ std::vector
Multicaster::getMembers(uint64_t nwid,const MulticastGroup void Multicaster::send( void *tPtr, unsigned int limit, - uint64_t now, + int64_t now, uint64_t nwid, bool disableCompression, const std::vector
&alwaysSendTo, @@ -309,7 +309,7 @@ void Multicaster::send( delete [] indexes; } -void Multicaster::clean(uint64_t now) +void Multicaster::clean(int64_t now) { { Mutex::Lock _l(_groups_m); @@ -367,7 +367,7 @@ void Multicaster::addCredential(void *tPtr,const CertificateOfMembership &com,bo } } -void Multicaster::_add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) +void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member) { // assumes _groups_m is locked diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 69a6645d..08c96485 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -98,7 +98,7 @@ public: * @param mg Multicast group * @param member New member address */ - inline void add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) + inline void add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &member) { Mutex::Lock _l(_groups_m); _add(tPtr,now,nwid,mg,_groups[Multicaster::Key(nwid,mg)],member); @@ -117,7 +117,7 @@ public: * @param count Number of addresses * @param totalKnown Total number of known addresses as reported by peer */ - void addMultiple(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); + void addMultiple(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,unsigned int totalKnown); /** * Remove a multicast group member (if present) @@ -174,7 +174,7 @@ public: void send( void *tPtr, unsigned int limit, - uint64_t now, + int64_t now, uint64_t nwid, bool disableCompression, const std::vector
&alwaysSendTo, @@ -190,7 +190,7 @@ public: * @param RR Runtime environment * @param now Current time */ - void clean(uint64_t now); + void clean(int64_t now); /** * Add an authorization credential @@ -212,7 +212,7 @@ public: * @param now Current time * @return True if GATHER and LIKE should be allowed */ - bool cacheAuthorized(const Address &a,const uint64_t nwid,const uint64_t now) const + bool cacheAuthorized(const Address &a,const uint64_t nwid,const int64_t now) const { Mutex::Lock _l(_gatherAuth_m); const uint64_t *p = _gatherAuth.get(_GatherAuthKey(nwid,a)); @@ -220,7 +220,7 @@ public: } private: - void _add(void *tPtr,uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); + void _add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member); const RuntimeEnvironment *RR; diff --git a/node/Network.cpp b/node/Network.cpp index 16155c33..111e4736 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -592,7 +592,7 @@ bool Network::filterOutgoingPacket( const unsigned int etherType, const unsigned int vlanId) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); Address ztFinalDest(ztDest); int localCapabilityIndex = -1; int accept = 0; @@ -1164,7 +1164,7 @@ void Network::requestConfiguration(void *tPtr) bool Network::gate(void *tPtr,const SharedPtr &peer) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); Mutex::Lock _l(_lock); try { if (_config) { @@ -1192,7 +1192,7 @@ bool Network::recentlyAssociatedWith(const Address &addr) void Network::clean() { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); Mutex::Lock _l(_lock); if (_destroyed) @@ -1257,7 +1257,7 @@ void Network::learnBridgeRoute(const MAC &mac,const Address &addr) } } -void Network::learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now) +void Network::learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,int64_t now) { Mutex::Lock _l(_lock); const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size(); @@ -1377,7 +1377,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup) { // Assumes _lock is locked - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); std::vector groups; if (newMulticastGroup) diff --git a/node/Network.hpp b/node/Network.hpp index d4d217f2..1b4da7d2 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -305,7 +305,7 @@ public: * @param mg Multicast group * @param now Current time */ - void learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,uint64_t now); + void learnBridgedMulticastGroup(void *tPtr,const MulticastGroup &mg,int64_t now); /** * Validate a credential and learn it if it passes certificate and other checks @@ -357,7 +357,7 @@ public: * @param to Destination peer address * @param now Current time */ - inline void pushCredentialsNow(void *tPtr,const Address &to,const uint64_t now) + inline void pushCredentialsNow(void *tPtr,const Address &to,const int64_t now) { Mutex::Lock _l(_lock); _membership(to).pushCredentials(RR,tPtr,now,to,_config,-1,true); diff --git a/node/Node.cpp b/node/Node.cpp index cc076e4d..31ee8f19 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -54,7 +54,7 @@ namespace ZeroTier { /* Public Node interface (C++, exposed via CAPI bindings) */ /****************************************************************************/ -Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) : +Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now) : _RR(this), RR(&_RR), _uPtr(uptr), @@ -139,12 +139,12 @@ Node::~Node() ZT_ResultCode Node::processWirePacket( void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { _now = now; RR->sw->onRemotePacket(tptr,localSocket,*(reinterpret_cast(remoteAddress)),packetData,packetLength); @@ -153,7 +153,7 @@ ZT_ResultCode Node::processWirePacket( ZT_ResultCode Node::processVirtualNetworkFrame( void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -161,7 +161,7 @@ ZT_ResultCode Node::processVirtualNetworkFrame( unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { _now = now; SharedPtr nw(this->network(nwid)); @@ -175,7 +175,7 @@ ZT_ResultCode Node::processVirtualNetworkFrame( class _PingPeersThatNeedPing { public: - _PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector > &upstreamsToContact,uint64_t now) : + _PingPeersThatNeedPing(const RuntimeEnvironment *renv,void *tPtr,Hashtable< Address,std::vector > &upstreamsToContact,int64_t now) : lastReceiveFromUpstream(0), RR(renv), _tPtr(tPtr), @@ -185,7 +185,7 @@ public: { } - uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay + int64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay inline void operator()(Topology &t,const SharedPtr &p) { @@ -234,17 +234,17 @@ private: const RuntimeEnvironment *RR; void *_tPtr; Hashtable< Address,std::vector > &_upstreamsToContact; - const uint64_t _now; + const int64_t _now; const SharedPtr _bestCurrentUpstream; }; -ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline) { _now = now; Mutex::Lock bl(_backgroundTasksLock); unsigned long timeUntilNextPingCheck = ZT_PING_CHECK_INVERVAL; - const uint64_t timeSinceLastPingCheck = now - _lastPingCheck; + const int64_t timeSinceLastPingCheck = now - _lastPingCheck; if (timeSinceLastPingCheck >= ZT_PING_CHECK_INVERVAL) { try { _lastPingCheck = now; @@ -305,7 +305,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint } try { - *nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); + *nextBackgroundTaskDeadline = now + (int64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(tptr,now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY); } catch ( ... ) { return ZT_RESULT_FATAL_ERROR_INTERNAL; } @@ -689,7 +689,7 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des extern "C" { -enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) +enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now) { *node = (ZT_Node *)0; try { @@ -714,12 +714,12 @@ void ZT_Node_delete(ZT_Node *node) enum ZT_ResultCode ZT_Node_processWirePacket( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { try { return reinterpret_cast(node)->processWirePacket(tptr,now,localSocket,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline); @@ -733,7 +733,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket( enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( ZT_Node *node, void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -741,7 +741,7 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline) + volatile int64_t *nextBackgroundTaskDeadline) { try { return reinterpret_cast(node)->processVirtualNetworkFrame(tptr,now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline); @@ -752,7 +752,7 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame( } } -enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline) +enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline) { try { return reinterpret_cast(node)->processBackgroundTasks(tptr,now,nextBackgroundTaskDeadline); diff --git a/node/Node.hpp b/node/Node.hpp index 1aa01c9a..ae7976d4 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -64,7 +64,7 @@ class World; class Node : public NetworkController::Sender { public: - Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now); + Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64_t now); virtual ~Node(); // Get rid of alignment warnings on 32-bit Windows and possibly improve performance @@ -77,15 +77,15 @@ public: ZT_ResultCode processWirePacket( void *tptr, - uint64_t now, + int64_t now, int64_t localSocket, const struct sockaddr_storage *remoteAddress, const void *packetData, unsigned int packetLength, - volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); ZT_ResultCode processVirtualNetworkFrame( void *tptr, - uint64_t now, + int64_t now, uint64_t nwid, uint64_t sourceMac, uint64_t destMac, @@ -93,8 +93,8 @@ public: unsigned int vlanId, const void *frameData, unsigned int frameLength, - volatile uint64_t *nextBackgroundTaskDeadline); - ZT_ResultCode processBackgroundTasks(void *tptr,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline); + volatile int64_t *nextBackgroundTaskDeadline); + ZT_ResultCode processBackgroundTasks(void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline); ZT_ResultCode join(uint64_t nwid,void *uptr,void *tptr); ZT_ResultCode leave(uint64_t nwid,void **uptr,void *tptr); ZT_ResultCode multicastSubscribe(void *tptr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi); @@ -114,7 +114,7 @@ public: // Internal functions ------------------------------------------------------ - inline uint64_t now() const { return _now; } + inline int64_t now() const { return _now; } inline bool putPacket(void *tPtr,const int64_t localSocket,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0) { @@ -243,7 +243,7 @@ public: * @param from Source address of packet * @return True if within rate limits */ - inline bool rateGateIdentityVerification(const uint64_t now,const InetAddress &from) + inline bool rateGateIdentityVerification(const int64_t now,const InetAddress &from) { unsigned long iph = from.rateGateHash(); if ((now - _lastIdentityVerification[iph]) >= ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT) { @@ -270,7 +270,7 @@ private: uint32_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1]; // Time of last identity verification indexed by InetAddress.rateGateHash() -- used in IncomingPacket::_doHELLO() via rateGateIdentityVerification() - uint64_t _lastIdentityVerification[16384]; + int64_t _lastIdentityVerification[16384]; Hashtable< uint64_t,SharedPtr > _networks; Mutex _networks_m; @@ -281,10 +281,10 @@ private: Mutex _backgroundTasksLock; Address _remoteTraceTarget; - uint64_t _now; - uint64_t _lastPingCheck; - uint64_t _lastHousekeepingRun; - volatile uint64_t _prngState[2]; + int64_t _now; + int64_t _lastPingCheck; + int64_t _lastHousekeepingRun; + volatile int64_t _prngState[2]; bool _online; }; diff --git a/node/OutboundMulticast.hpp b/node/OutboundMulticast.hpp index 486b66ff..2f6d8338 100644 --- a/node/OutboundMulticast.hpp +++ b/node/OutboundMulticast.hpp @@ -96,7 +96,7 @@ public: * @param now Current time * @return True if this multicast is expired (has exceeded transmit timeout) */ - inline bool expired(uint64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); } + inline bool expired(int64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); } /** * @return True if this outbound multicast has been sent to enough peers diff --git a/node/Path.cpp b/node/Path.cpp index 9dc9aba5..ca366e39 100644 --- a/node/Path.cpp +++ b/node/Path.cpp @@ -30,7 +30,7 @@ namespace ZeroTier { -bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now) +bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now) { if (RR->node->putPacket(tPtr,_localSocket,_addr,data,len)) { _lastOut = now; diff --git a/node/Path.hpp b/node/Path.hpp index ac8e4c0e..050fb6e2 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -179,14 +179,14 @@ public: * @param now Current time * @return True if transport reported success */ - bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,uint64_t now); + bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now); /** * Manually update last sent time * * @param t Time of send */ - inline void sent(const uint64_t t) { _lastOut = t; } + inline void sent(const int64_t t) { _lastOut = t; } /** * @return Local socket as specified by external code @@ -206,7 +206,7 @@ public: /** * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms */ - inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } + inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } /** * @return Preference rank, higher == better @@ -261,27 +261,27 @@ public: /** * @return True if path appears alive */ - inline bool alive(const uint64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); } + inline bool alive(const int64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); } /** * @return True if this path needs a heartbeat */ - inline bool needsHeartbeat(const uint64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); } + inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); } /** * @return Last time we sent something */ - inline uint64_t lastOut() const { return _lastOut; } + inline int64_t lastOut() const { return _lastOut; } /** * @return Last time we received anything */ - inline uint64_t lastIn() const { return _lastIn; } + inline int64_t lastIn() const { return _lastIn; } /** * @return Time last trust-established packet was received */ - inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } + inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } /** * Return and increment outgoing packet counter (used with Packet::armor()) @@ -291,9 +291,9 @@ public: inline unsigned int nextOutgoingCounter() { return _outgoingPacketCounter++; } private: - volatile uint64_t _lastOut; - volatile uint64_t _lastIn; - volatile uint64_t _lastTrustEstablishedPacketReceived; + volatile int64_t _lastOut; + volatile int64_t _lastIn; + volatile int64_t _lastTrustEstablishedPacketReceived; volatile uint64_t _incomingLinkQualityFastLog; int64_t _localSocket; volatile unsigned long _incomingLinkQualitySlowLogPtr; diff --git a/node/Peer.cpp b/node/Peer.cpp index a954b716..60661592 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -34,6 +34,7 @@ #include "SelfAwareness.hpp" #include "Packet.hpp" #include "Trace.hpp" +#include "InetAddress.hpp" namespace ZeroTier { @@ -75,7 +76,7 @@ void Peer::received( const bool trustEstablished, const uint64_t networkId) { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); /* #ifdef ZT_ENABLE_CLUSTER @@ -263,14 +264,14 @@ void Peer::received( } } -bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) +bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force) { Mutex::Lock _l(_paths_m); - uint64_t v6lr = 0; + int64_t v6lr = 0; if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) v6lr = _v6Path.p->lastIn(); - uint64_t v4lr = 0; + int64_t v4lr = 0; if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) v4lr = _v4Path.p->lastIn(); @@ -289,16 +290,18 @@ bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now, return false; } -SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) +SharedPtr Peer::getBestPath(int64_t now,bool includeExpired) { Mutex::Lock _l(_paths_m); - uint64_t v6lr = 0; - if ( ( includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v6Path.p) ) + int64_t v6lr = 0; + if ((includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)) && (_v6Path.p)) { v6lr = _v6Path.p->lastIn(); - uint64_t v4lr = 0; - if ( ( includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v4Path.p) ) + } + int64_t v4lr = 0; + if ((includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)) && (_v4Path.p)) { v4lr = _v4Path.p->lastIn(); + } if (v6lr > v4lr) { return _v6Path.p; @@ -309,7 +312,7 @@ SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) return SharedPtr(); } -void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,unsigned int counter) +void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,unsigned int counter) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO); @@ -357,7 +360,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA } } -void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter) +void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,bool sendFullHello,unsigned int counter) { if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) { Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO); @@ -369,7 +372,7 @@ void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAdd } } -void Peer::tryMemorizedPath(void *tPtr,uint64_t now) +void Peer::tryMemorizedPath(void *tPtr,int64_t now) { if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) { _lastTriedMemorizedPath = now; @@ -379,15 +382,15 @@ void Peer::tryMemorizedPath(void *tPtr,uint64_t now) } } -bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) +bool Peer::doPingAndKeepalive(void *tPtr,int64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); if (inetAddressFamily < 0) { - uint64_t v6lr = 0; + int64_t v6lr = 0; if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) v6lr = _v6Path.p->lastIn(); - uint64_t v4lr = 0; + int64_t v4lr = 0; if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) v4lr = _v4Path.p->lastIn(); @@ -423,7 +426,7 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } -void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now) +void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now) { if ((remoteAddress.ss_family != AF_INET)&&(remoteAddress.ss_family != AF_INET6)) // sanity check return; diff --git a/node/Peer.hpp b/node/Peer.hpp index af9163a5..e08f7d36 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -120,7 +120,7 @@ public: * @param addr Remote address * @return True if we have an active path to this destination */ - inline bool hasActivePathTo(uint64_t now,const InetAddress &addr) const + inline bool hasActivePathTo(int64_t now,const InetAddress &addr) const { Mutex::Lock _l(_paths_m); return ( ((addr.ss_family == AF_INET)&&(_v4Path.p)&&(_v4Path.p->address() == addr)&&(_v4Path.p->alive(now))) || ((addr.ss_family == AF_INET6)&&(_v6Path.p)&&(_v6Path.p->address() == addr)&&(_v6Path.p->alive(now))) ); @@ -136,7 +136,7 @@ public: * @param force If true, send even if path is not alive * @return True if we actually sent something */ - bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force); + bool sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force); /** * Get the best current direct path @@ -148,7 +148,7 @@ public: * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none */ - SharedPtr getBestPath(uint64_t now,bool includeExpired); + SharedPtr getBestPath(int64_t now,bool includeExpired); /** * Send a HELLO to this peer at a specified physical address @@ -161,7 +161,7 @@ public: * @param now Current time * @param counter Outgoing packet counter */ - void sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,unsigned int counter); + void sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,unsigned int counter); /** * Send ECHO (or HELLO for older peers) to this peer at the given address @@ -175,7 +175,7 @@ public: * @param sendFullHello If true, always send a full HELLO instead of just an ECHO * @param counter Outgoing packet counter */ - void attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,uint64_t now,bool sendFullHello,unsigned int counter); + void attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,bool sendFullHello,unsigned int counter); /** * Try a memorized or statically defined path if any are known @@ -185,7 +185,7 @@ public: * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time */ - void tryMemorizedPath(void *tPtr,uint64_t now); + void tryMemorizedPath(void *tPtr,int64_t now); /** * Send pings or keepalives depending on configured timeouts @@ -195,7 +195,7 @@ public: * @param inetAddressFamily Keep this address family alive, or -1 for any * @return True if we have at least one direct path of the given family (or any if family is -1) */ - bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); + bool doPingAndKeepalive(void *tPtr,int64_t now,int inetAddressFamily); /** * Specify remote path for this peer and forget others @@ -209,7 +209,7 @@ public: * @param remoteAddress Remote address * @param now Current time */ - void redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const uint64_t now); + void redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now); /** * Reset paths within a given IP scope and address family @@ -222,7 +222,7 @@ public: * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) + inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now) { Mutex::Lock _l(_paths_m); if ((inetAddressFamily == AF_INET)&&(_v4Path.lr)&&(_v4Path.p->address().ipScope() == scope)) { @@ -243,7 +243,7 @@ public: * @param v4 Result parameter to receive active IPv4 address, if any * @param v6 Result parameter to receive active IPv6 address, if any */ - inline void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const + inline void getRendezvousAddresses(int64_t now,InetAddress &v4,InetAddress &v6) const { Mutex::Lock _l(_paths_m); if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) @@ -256,7 +256,7 @@ public: * @param now Current time * @return All known paths to this peer */ - inline std::vector< SharedPtr > paths(const uint64_t now) const + inline std::vector< SharedPtr > paths(const int64_t now) const { std::vector< SharedPtr > pp; Mutex::Lock _l(_paths_m); @@ -270,17 +270,17 @@ public: /** * @return Time of last receive of anything, whether direct or relayed */ - inline uint64_t lastReceive() const { return _lastReceive; } + inline int64_t lastReceive() const { return _lastReceive; } /** * @return True if we've heard from this peer in less than ZT_PEER_ACTIVITY_TIMEOUT */ - inline bool isAlive(const uint64_t now) const { return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline bool isAlive(const int64_t now) const { return ((now - _lastReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return True if this peer has sent us real network traffic recently */ - inline uint64_t isActive(uint64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } + inline int64_t isActive(int64_t now) const { return ((now - _lastNontrivialReceive) < ZT_PEER_ACTIVITY_TIMEOUT); } /** * @return Latency in milliseconds or 0 if unknown @@ -298,7 +298,7 @@ public: * * @return Relay quality score computed from latency and other factors, lower is better */ - inline unsigned int relayQuality(const uint64_t now) const + inline unsigned int relayQuality(const int64_t now) const { const uint64_t tsr = now - _lastReceive; if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT) @@ -353,12 +353,12 @@ public: /** * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms */ - inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } + inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); } /** * Rate limit gate for VERB_PUSH_DIRECT_PATHS */ - inline bool rateGatePushDirectPaths(const uint64_t now) + inline bool rateGatePushDirectPaths(const int64_t now) { if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME) ++_directPathPushCutoffCount; @@ -370,7 +370,7 @@ public: /** * Rate limit gate for VERB_NETWORK_CREDENTIALS */ - inline bool rateGateCredentialsReceived(const uint64_t now) + inline bool rateGateCredentialsReceived(const int64_t now) { if ((now - _lastCredentialsReceived) <= ZT_PEER_CREDENTIALS_CUTOFF_TIME) ++_credentialsCutoffCount; @@ -382,7 +382,7 @@ public: /** * Rate limit gate for sending of ERROR_NEED_MEMBERSHIP_CERTIFICATE */ - inline bool rateGateRequestCredentials(const uint64_t now) + inline bool rateGateRequestCredentials(const int64_t now) { if ((now - _lastCredentialRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastCredentialRequestSent = now; @@ -394,7 +394,7 @@ public: /** * Rate limit gate for inbound WHOIS requests */ - inline bool rateGateInboundWhoisRequest(const uint64_t now) + inline bool rateGateInboundWhoisRequest(const int64_t now) { if ((now - _lastWhoisRequestReceived) >= ZT_PEER_WHOIS_RATE_LIMIT) { _lastWhoisRequestReceived = now; @@ -406,7 +406,7 @@ public: /** * Rate limit gate for inbound ECHO requests */ - inline bool rateGateEchoRequest(const uint64_t now) + inline bool rateGateEchoRequest(const int64_t now) { if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastEchoRequestReceived = now; @@ -418,7 +418,7 @@ public: /** * Rate gate incoming requests for network COM */ - inline bool rateGateIncomingComRequest(const uint64_t now) + inline bool rateGateIncomingComRequest(const int64_t now) { if ((now - _lastComRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastComRequestReceived = now; @@ -430,7 +430,7 @@ public: /** * Rate gate outgoing requests for network COM */ - inline bool rateGateOutgoingComRequest(const uint64_t now) + inline bool rateGateOutgoingComRequest(const int64_t now) { if ((now - _lastComRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) { _lastComRequestSent = now; @@ -484,7 +484,7 @@ public: } template - inline static SharedPtr deserializeFromCache(uint64_t now,void *tPtr,Buffer &b,const RuntimeEnvironment *renv) + inline static SharedPtr deserializeFromCache(int64_t now,void *tPtr,Buffer &b,const RuntimeEnvironment *renv) { try { unsigned int ptr = 0; @@ -527,8 +527,8 @@ private: struct _PeerPath { _PeerPath() : lr(0),sticky(0),p() {} - uint64_t lr; // time of last valid ZeroTier packet - uint64_t sticky; // time last set as sticky + int64_t lr; // time of last valid ZeroTier packet + int64_t sticky; // time last set as sticky SharedPtr p; }; @@ -536,18 +536,18 @@ private: const RuntimeEnvironment *RR; - uint64_t _lastReceive; // direct or indirect - uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. - uint64_t _lastTriedMemorizedPath; - uint64_t _lastDirectPathPushSent; - uint64_t _lastDirectPathPushReceive; - uint64_t _lastCredentialRequestSent; - uint64_t _lastWhoisRequestReceived; - uint64_t _lastEchoRequestReceived; - uint64_t _lastComRequestReceived; - uint64_t _lastComRequestSent; - uint64_t _lastCredentialsReceived; - uint64_t _lastTrustEstablishedPacketReceived; + int64_t _lastReceive; // direct or indirect + int64_t _lastNontrivialReceive; // frames, things like netconf, etc. + int64_t _lastTriedMemorizedPath; + int64_t _lastDirectPathPushSent; + int64_t _lastDirectPathPushReceive; + int64_t _lastCredentialRequestSent; + int64_t _lastWhoisRequestReceived; + int64_t _lastEchoRequestReceived; + int64_t _lastComRequestReceived; + int64_t _lastComRequestSent; + int64_t _lastCredentialsReceived; + int64_t _lastTrustEstablishedPacketReceived; uint16_t _vProto; uint16_t _vMajor; diff --git a/node/Revocation.hpp b/node/Revocation.hpp index a28da0ab..7f7498bb 100644 --- a/node/Revocation.hpp +++ b/node/Revocation.hpp @@ -85,7 +85,7 @@ public: inline uint32_t id() const { return _id; } inline uint32_t credentialId() const { return _credentialId; } inline uint64_t networkId() const { return _networkId; } - inline uint64_t threshold() const { return _threshold; } + inline int64_t threshold() const { return _threshold; } inline const Address &target() const { return _target; } inline const Address &signer() const { return _signedBy; } inline Credential::Type type() const { return _type; } @@ -184,7 +184,7 @@ private: uint32_t _id; uint32_t _credentialId; uint64_t _networkId; - uint64_t _threshold; + int64_t _threshold; uint64_t _flags; Address _target; Address _signedBy; diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 0af0d691..83cd89c9 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -49,7 +49,7 @@ namespace ZeroTier { class _ResetWithinScope { public: - _ResetWithinScope(void *tPtr,uint64_t now,int inetAddressFamily,InetAddress::IpScope scope) : + _ResetWithinScope(void *tPtr,int64_t now,int inetAddressFamily,InetAddress::IpScope scope) : _now(now), _tPtr(tPtr), _family(inetAddressFamily), @@ -70,7 +70,7 @@ SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) : { } -void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now) +void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,int64_t now) { const InetAddress::IpScope scope = myPhysicalAddress.ipScope(); @@ -112,7 +112,7 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receive } } -void SelfAwareness::clean(uint64_t now) +void SelfAwareness::clean(int64_t now) { Mutex::Lock _l(_phy_m); Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index 35e0ad39..7ddba465 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -55,14 +55,14 @@ public: * @param trusted True if this peer is trusted as an authority to inform us of external address changes * @param now Current time */ - void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now); + void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,int64_t now); /** * Clean up database periodically * * @param now Current time */ - void clean(uint64_t now); + void clean(int64_t now); /** * If we appear to be behind a symmetric NAT, get predictions for possible external endpoints diff --git a/node/Switch.cpp b/node/Switch.cpp index f46b3e73..cc022b6b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -57,7 +57,7 @@ Switch::Switch(const RuntimeEnvironment *renv) : void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddress &fromAddr,const void *data,unsigned int len) { try { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); const SharedPtr path(RR->topology->getPath(localSocket,fromAddr)); path->received(now); @@ -557,14 +557,14 @@ void Switch::send(void *tPtr,Packet &packet,bool encrypt) } } -void Switch::requestWhois(void *tPtr,const uint64_t now,const Address &addr) +void Switch::requestWhois(void *tPtr,const int64_t now,const Address &addr) { if (addr == RR->identity.address()) return; { Mutex::Lock _l(_lastSentWhoisRequest_m); - uint64_t &last = _lastSentWhoisRequest[addr]; + int64_t &last = _lastSentWhoisRequest[addr]; if ((now - last) < ZT_WHOIS_RETRY_DELAY) return; else last = now; @@ -586,7 +586,7 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) _lastSentWhoisRequest.erase(peer->address()); } - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); for(unsigned int ptr=0;ptrtimestamp)&&(rq->complete)) { @@ -611,7 +611,7 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr &peer) } } -unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) +unsigned long Switch::doTimerTasks(void *tPtr,int64_t now) { const uint64_t timeSinceLastCheck = now - _lastCheckedQueues; if (timeSinceLastCheck < ZT_WHOIS_RETRY_DELAY) @@ -663,9 +663,9 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) { Mutex::Lock _l(_lastSentWhoisRequest_m); - Hashtable< Address,uint64_t >::Iterator i(_lastSentWhoisRequest); + Hashtable< Address,int64_t >::Iterator i(_lastSentWhoisRequest); Address *a = (Address *)0; - uint64_t *ts = (uint64_t *)0; + int64_t *ts = (int64_t *)0; while (i.next(a,ts)) { if ((now - *ts) > (ZT_WHOIS_RETRY_DELAY * 2)) _lastSentWhoisRequest.erase(*a); @@ -675,7 +675,7 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now) return ZT_WHOIS_RETRY_DELAY; } -bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address &destination) +bool Switch::_shouldUnite(const int64_t now,const Address &source,const Address &destination) { Mutex::Lock _l(_lastUniteAttempt_m); uint64_t &ts = _lastUniteAttempt[_LastUniteKey(source,destination)]; @@ -689,7 +689,7 @@ bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) { SharedPtr viaPath; - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); const Address destination(packet.destination()); const SharedPtr peer(RR->topology->getPeer(tPtr,destination)); @@ -703,7 +703,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) viaPath = peer->getBestPath(now,false); if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { - if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { + if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(int64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { peer->attemptToContactAt(tPtr,viaPath->localSocket(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); viaPath->sent(now); } diff --git a/node/Switch.hpp b/node/Switch.hpp index c258a255..b42389fc 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -114,7 +114,7 @@ public: * @param now Current time * @param addr Address to look up */ - void requestWhois(void *tPtr,const uint64_t now,const Address &addr); + void requestWhois(void *tPtr,const int64_t now,const Address &addr); /** * Run any processes that are waiting for this peer's identity @@ -136,25 +136,25 @@ public: * @param now Current time * @return Number of milliseconds until doTimerTasks() should be run again */ - unsigned long doTimerTasks(void *tPtr,uint64_t now); + unsigned long doTimerTasks(void *tPtr,int64_t now); private: - bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination); + bool _shouldUnite(const int64_t now,const Address &source,const Address &destination); bool _trySend(void *tPtr,Packet &packet,bool encrypt); // packet is modified if return is true const RuntimeEnvironment *const RR; - uint64_t _lastBeaconResponse; - volatile uint64_t _lastCheckedQueues; + int64_t _lastBeaconResponse; + volatile int64_t _lastCheckedQueues; // Time we last sent a WHOIS request for each address - Hashtable< Address,uint64_t > _lastSentWhoisRequest; + Hashtable< Address,int64_t > _lastSentWhoisRequest; Mutex _lastSentWhoisRequest_m; // Packets waiting for WHOIS replies or other decode info or missing fragments struct RXQueueEntry { RXQueueEntry() : timestamp(0) {} - volatile uint64_t timestamp; // 0 if entry is not in use + volatile int64_t timestamp; // 0 if entry is not in use volatile uint64_t packetId; IncomingPacket frag0; // head of packet Packet::Fragment frags[ZT_MAX_PACKET_FRAGMENTS - 1]; // later fragments (if any) diff --git a/node/Topology.cpp b/node/Topology.cpp index 8a830b93..f884e9c3 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -133,8 +133,9 @@ SharedPtr Topology::getPeer(void *tPtr,const Address &zta) if (ap) return ap; ap = Peer::deserializeFromCache(RR->node->now(),tPtr,buf,RR); - if (!ap) + if (!ap) { _peers.erase(zta); + } return SharedPtr(); } } catch ( ... ) {} // ignore invalid identities or other strage failures @@ -157,7 +158,7 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta) SharedPtr Topology::getUpstreamPeer() { - const uint64_t now = RR->node->now(); + const int64_t now = RR->node->now(); unsigned int bestq = ~((unsigned int)0); const SharedPtr *best = (const SharedPtr *)0; @@ -365,7 +366,7 @@ void Topology::removeMoon(void *tPtr,const uint64_t id) _memoizeUpstreams(tPtr); } -void Topology::doPeriodicTasks(void *tPtr,uint64_t now) +void Topology::doPeriodicTasks(void *tPtr,int64_t now) { { Mutex::Lock _l1(_peers_m); diff --git a/node/Topology.hpp b/node/Topology.hpp index 34df28a1..c3a218e3 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -286,13 +286,13 @@ public: /** * Clean and flush database */ - void doPeriodicTasks(void *tPtr,uint64_t now); + void doPeriodicTasks(void *tPtr,int64_t now); /** * @param now Current time * @return Number of peers with active direct paths */ - inline unsigned long countActive(uint64_t now) const + inline unsigned long countActive(int64_t now) const { unsigned long cnt = 0; Mutex::Lock _l(_peers_m); diff --git a/one.cpp b/one.cpp index b1a19e8c..de16cc2d 100644 --- a/one.cpp +++ b/one.cpp @@ -364,9 +364,9 @@ static int cli(int argc,char **argv) if (path["preferred"]) { char tmp[256]; std::string addr = path["address"]; - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); const double lq = (path.count("linkQuality")) ? (double)path["linkQuality"] : -1.0; - OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%llu;%llu;%1.2f",addr.c_str(),now - (uint64_t)path["lastSend"],now - (uint64_t)path["lastReceive"],lq); + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%lld;%lld;%1.2f",addr.c_str(),now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"],lq); bestPath = tmp; break; } @@ -864,7 +864,7 @@ static int idtool(int argc,char **argv) } std::sort(roots.begin(),roots.end()); - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); World w(World::make(t,id,now,updatesMustBeSignedBy,roots,signingKey)); Buffer wbuf; w.serialize(wbuf); diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 8683ba25..8f66f850 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -200,7 +200,7 @@ public: /** * @return Current time in milliseconds since epoch */ - static inline uint64_t now() + static inline int64_t now() { #ifdef __WINDOWS__ FILETIME ft; @@ -210,7 +210,7 @@ public: SystemTimeToFileTime(&st,&ft); tmp.LowPart = ft.dwLowDateTime; tmp.HighPart = ft.dwHighDateTime; - return ( ((tmp.QuadPart - 116444736000000000ULL) / 10000L) + st.wMilliseconds ); + return (int64_t)( ((tmp.QuadPart - 116444736000000000LL) / 10000L) + st.wMilliseconds ); #else struct timeval tv; #ifdef __LINUX__ @@ -218,7 +218,7 @@ public: #else gettimeofday(&tv,(struct timezone *)0); #endif - return ( (1000ULL * (uint64_t)tv.tv_sec) + (uint64_t)(tv.tv_usec / 1000) ); + return ( (1000LL * (int64_t)tv.tv_sec) + (int64_t)(tv.tv_usec / 1000) ); #endif }; diff --git a/osdep/PortMapper.cpp b/osdep/PortMapper.cpp index 0da00653..825972b0 100644 --- a/osdep/PortMapper.cpp +++ b/osdep/PortMapper.cpp @@ -131,7 +131,7 @@ public: InetAddress publicAddress; sendpublicaddressrequest(&natpmp); - uint64_t myTimeout = OSUtils::now() + 5000; + int64_t myTimeout = OSUtils::now() + 5000; do { fd_set fds; struct timeval timeout; diff --git a/service/OneService.cpp b/service/OneService.cpp index 66e9a9c8..fb185ee7 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -276,6 +276,10 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) pa.push_back(j); } pj["paths"] = pa; + + if (peer->address == 0xda6c71a1ad) { + fprintf(stdout, "%s\n", pj.dump(2).c_str()); + } } static void _moonToJson(nlohmann::json &mj,const World &world) @@ -436,7 +440,7 @@ public: uint64_t _lastRestart; // Deadline for the next background task service function - volatile uint64_t _nextBackgroundTaskDeadline; + volatile int64_t _nextBackgroundTaskDeadline; // Configured networks struct NetworkState @@ -755,12 +759,12 @@ public: // Main I/O loop _nextBackgroundTaskDeadline = 0; - uint64_t clockShouldBe = OSUtils::now(); + int64_t clockShouldBe = OSUtils::now(); _lastRestart = clockShouldBe; - uint64_t lastTapMulticastGroupCheck = 0; - uint64_t lastBindRefresh = 0; - uint64_t lastUpdateCheck = clockShouldBe; - uint64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle + int64_t lastTapMulticastGroupCheck = 0; + int64_t lastBindRefresh = 0; + int64_t lastUpdateCheck = clockShouldBe; + int64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle for(;;) { _run_m.lock(); if (!_run) { @@ -773,7 +777,7 @@ public: _run_m.unlock(); } - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); // Attempt to detect sleep/wake events by detecting delay overruns bool restarted = false; @@ -809,7 +813,7 @@ public: } // Run background task processor in core if it's time to do so - uint64_t dl = _nextBackgroundTaskDeadline; + int64_t dl = _nextBackgroundTaskDeadline; if (dl <= now) { _node->processBackgroundTasks((void *)0,now,&_nextBackgroundTaskDeadline); dl = _nextBackgroundTaskDeadline; @@ -2152,7 +2156,7 @@ public: // Engage TCP tunnel fallback if we haven't received anything valid from a global // IP address in ZT_TCP_FALLBACK_AFTER milliseconds. If we do start getting // valid direct traffic we'll stop using it and close the socket after a while. - const uint64_t now = OSUtils::now(); + const int64_t now = OSUtils::now(); if (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER)) { if (_tcpFallbackTunnel) { bool flushNow = false; diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 11005945..39833c90 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -303,7 +303,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void } } -bool SoftwareUpdater::check(const uint64_t now) +bool SoftwareUpdater::check(const int64_t now) { if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index ff3e36df..f16c99a0 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -167,7 +167,7 @@ public: * * @return True if we've downloaded and verified an update */ - bool check(const uint64_t now); + bool check(const int64_t now); /** * @return Meta-data for downloaded update or NULL if none -- cgit v1.2.3 From 099bedd2e925a489d59fd4ecca59813944887f5f Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Wed, 4 Oct 2017 12:01:17 -0700 Subject: A few more uint64_t -> int64_t changes for timestamps --- node/Capability.hpp | 6 +++--- node/CertificateOfOwnership.hpp | 6 +++--- node/Membership.cpp | 6 +++--- node/Membership.hpp | 6 +++--- node/NetworkConfig.hpp | 4 ++-- node/Tag.hpp | 6 +++--- 6 files changed, 17 insertions(+), 17 deletions(-) (limited to 'node') diff --git a/node/Capability.hpp b/node/Capability.hpp index d64625e6..c0336243 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -85,7 +85,7 @@ public: * @param rules Network flow rules for this capability * @param ruleCount Number of flow rules */ - Capability(uint32_t id,uint64_t nwid,uint64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) + Capability(uint32_t id,uint64_t nwid,int64_t ts,unsigned int mccl,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) { memset(this,0,sizeof(Capability)); _nwid = nwid; @@ -120,7 +120,7 @@ public: /** * @return Timestamp */ - inline uint64_t timestamp() const { return _ts; } + inline int64_t timestamp() const { return _ts; } /** * @return Last 'to' address in chain of custody @@ -460,7 +460,7 @@ public: private: uint64_t _nwid; - uint64_t _ts; + int64_t _ts; uint32_t _id; unsigned int _maxCustodyChainLength; diff --git a/node/CertificateOfOwnership.hpp b/node/CertificateOfOwnership.hpp index 775324e6..431bcc03 100644 --- a/node/CertificateOfOwnership.hpp +++ b/node/CertificateOfOwnership.hpp @@ -72,7 +72,7 @@ public: memset(this,0,sizeof(CertificateOfOwnership)); } - CertificateOfOwnership(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id) : + CertificateOfOwnership(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id) : _networkId(nwid), _ts(ts), _flags(0), @@ -85,7 +85,7 @@ public: } inline uint64_t networkId() const { return _networkId; } - inline uint64_t timestamp() const { return _ts; } + inline int64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline unsigned int thingCount() const { return (unsigned int)_thingCount; } @@ -231,7 +231,7 @@ private: bool _owns(const Thing &t,const void *v,unsigned int l) const; uint64_t _networkId; - uint64_t _ts; + int64_t _ts; uint64_t _flags; uint32_t _id; uint16_t _thingCount; diff --git a/node/Membership.cpp b/node/Membership.cpp index 740f4e68..de5fb99d 100644 --- a/node/Membership.cpp +++ b/node/Membership.cpp @@ -155,7 +155,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme // Template out addCredential() for many cred types to avoid copypasta template -static Membership::AddCredentialResult _addCredImpl(Hashtable &remoteCreds,const Hashtable &revocations,const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const C &cred) +static Membership::AddCredentialResult _addCredImpl(Hashtable &remoteCreds,const Hashtable &revocations,const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const C &cred) { C *rc = remoteCreds.get(cred.id()); if (rc) { @@ -167,7 +167,7 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable &remot return Membership::ADD_ACCEPTED_REDUNDANT; } - const uint64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); + const int64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); if ((rt)&&(*rt >= cred.timestamp())) { RR->t->credentialRejected(tPtr,cred,"revoked"); return Membership::ADD_REJECTED; @@ -193,7 +193,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { - uint64_t *rt; + int64_t *rt; switch(rev.verify(RR,tPtr)) { default: RR->t->credentialRejected(tPtr,rev,"invalid"); diff --git a/node/Membership.hpp b/node/Membership.hpp index 5612858a..95ec2180 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -202,9 +202,9 @@ private: template inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &remoteCredential) const { - const uint64_t ts = remoteCredential.timestamp(); + const int64_t ts = remoteCredential.timestamp(); if (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) { - const uint64_t *threshold = _revocations.get(credentialKey(C::credentialType(),remoteCredential.id())); + const int64_t *threshold = _revocations.get(credentialKey(C::credentialType(),remoteCredential.id())); return ((!threshold)||(ts > *threshold)); } return false; @@ -235,7 +235,7 @@ private: CertificateOfMembership _com; // Revocations by credentialKey() - Hashtable< uint64_t,uint64_t > _revocations; + Hashtable< uint64_t,int64_t > _revocations; // Remote credentials that we have received from this member (and that are valid) Hashtable< uint32_t,Tag > _remoteTags; diff --git a/node/NetworkConfig.hpp b/node/NetworkConfig.hpp index 3fd5ddac..1bd3b9aa 100644 --- a/node/NetworkConfig.hpp +++ b/node/NetworkConfig.hpp @@ -418,12 +418,12 @@ public: /** * Controller-side time of config generation/issue */ - uint64_t timestamp; + int64_t timestamp; /** * Max difference between timestamp and tag/capability timestamp */ - uint64_t credentialTimeMaxDelta; + int64_t credentialTimeMaxDelta; /** * Controller-side revision counter for this configuration diff --git a/node/Tag.hpp b/node/Tag.hpp index 5fbfb278..fc1377de 100644 --- a/node/Tag.hpp +++ b/node/Tag.hpp @@ -77,7 +77,7 @@ public: * @param id Tag ID * @param value Tag value */ - Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : + Tag(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : _id(id), _value(value), _networkId(nwid), @@ -90,7 +90,7 @@ public: inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } inline uint64_t networkId() const { return _networkId; } - inline uint64_t timestamp() const { return _ts; } + inline int64_t timestamp() const { return _ts; } inline const Address &issuedTo() const { return _issuedTo; } inline const Address &signedBy() const { return _signedBy; } @@ -199,7 +199,7 @@ private: uint32_t _id; uint32_t _value; uint64_t _networkId; - uint64_t _ts; + int64_t _ts; Address _issuedTo; Address _signedBy; C25519::Signature _signature; -- cgit v1.2.3 From 395d8b31392309d2861cd2d50377aaa8953f42cd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 24 Oct 2017 13:33:53 -0700 Subject: Full and clearer implementation of GitHub issue #588 --- controller/EmbeddedNetworkController.cpp | 29 +++++++++++++++++++++++++++++ include/ZeroTierOne.h | 12 ++++++------ node/Capability.hpp | 13 +++++++++++++ node/Network.cpp | 29 +++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 6 deletions(-) (limited to 'node') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 20f81966..1a7cdbc7 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -227,6 +227,16 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) r["id"] = rule.v.tag.id; r["value"] = rule.v.tag.value; break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: + r["type"] = "INTEGER_RANGE"; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.intRange.start); + r["start"] = tmp; + OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",rule.v.intRange.start + (uint64_t)rule.v.intRange.end); + r["end"] = tmp; + r["idx"] = rule.v.intRange.idx; + r["little"] = ((rule.v.intRange.format & 0x80) != 0); + r["bits"] = (rule.v.intRange.format & 63) + 1; + break; default: break; } @@ -417,7 +427,26 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) } else if (t == "MATCH_TAG_RECEIVER") { rule.t |= ZT_NETWORK_RULE_MATCH_TAG_RECEIVER; tag = true; + } else if (t == "INTEGER_RANGE") { + json &s = r["start"]; + if (s.is_string()) { + std::string tmp = s; + rule.v.intRange.start = Utils::hexStrToU64(tmp.c_str()); + } else { + rule.v.intRange.start = OSUtils::jsonInt(s,0ULL); + } + json &e = r["end"]; + if (e.is_string()) { + std::string tmp = e; + rule.v.intRange.end = (uint32_t)(Utils::hexStrToU64(tmp.c_str()) - rule.v.intRange.start); + } else { + rule.v.intRange.end = (uint32_t)(OSUtils::jsonInt(e,0ULL) - rule.v.intRange.start); + } + rule.v.intRange.idx = (uint16_t)OSUtils::jsonInt(r["idx"],0ULL); + rule.v.intRange.format = (OSUtils::jsonBool(r["little"],false)) ? 0x80 : 0x00; + rule.v.intRange.format |= (uint8_t)((OSUtils::jsonInt(r["bits"],1ULL) - 1) & 63); } + if (tag) { rule.v.tag.id = (uint32_t)(OSUtils::jsonInt(r["id"],0ULL) & 0xffffffffULL); rule.v.tag.value = (uint32_t)(OSUtils::jsonInt(r["value"],0ULL) & 0xffffffffULL); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 8adbc4d1..02bb283b 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -52,13 +52,13 @@ #else #define ZT_SDK_API __declspec(dllimport) #ifdef _DEBUG -#ifdef _WIN64 +#ifdef _WIN64 #pragma comment(lib, "ZeroTierOne_x64d.lib") #else #pragma comment(lib, "ZeroTierOne_x86d.lib") #endif #else -#ifdef _WIN64 +#ifdef _WIN64 #pragma comment(lib, "ZeroTierOne_x64.lib") #else #pragma comment(lib, "ZeroTierOne_x86.lib") @@ -750,6 +750,7 @@ enum ZT_VirtualNetworkRuleType ZT_NETWORK_RULE_MATCH_TAGS_EQUAL = 48, ZT_NETWORK_RULE_MATCH_TAG_SENDER = 49, ZT_NETWORK_RULE_MATCH_TAG_RECEIVER = 50, + ZT_NETWORK_RULE_MATCH_INTEGER_RANGE = 51, /** * Maximum ID allowed for a MATCH entry in the rules table @@ -770,7 +771,7 @@ enum ZT_VirtualNetworkRuleType */ typedef struct { - /** + /** * Type and flags * * Bits are: NOTTTTTT @@ -812,10 +813,9 @@ typedef struct */ struct { uint64_t start; // integer range start - int32_t delta; // +/- offset from start of integer range end + uint32_t end; // end of integer range (relative to start, inclusive, 0 for equality w/start) uint16_t idx; // index in packet of integer - uint8_t bits; // number of bits in integer (range: 1-64) - uint8_t endian; // endianness of integer in packet (0 == big, 1 == little) + uint8_t format; // bits in integer (range 1-64, ((format&63)+1)) and endianness (MSB 1 for little, 0 for big) } intRange; /** diff --git a/node/Capability.hpp b/node/Capability.hpp index c0336243..407884ad 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -278,6 +278,13 @@ public: b.append((uint32_t)rules[i].v.tag.id); b.append((uint32_t)rules[i].v.tag.value); break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: + b.append((uint8_t)19); + b.append((uint64_t)rules[i].v.intRange.start); + b.append((uint64_t)(rules[i].v.intRange.start + (uint64_t)rules[i].v.intRange.end)); // more future-proof + b.append((uint16_t)rules[i].v.intRange.idx); + b.append((uint8_t)rules[i].v.intRange.format); + break; } } } @@ -366,6 +373,12 @@ public: rules[ruleCount].v.tag.id = b.template at(p); rules[ruleCount].v.tag.value = b.template at(p + 4); break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: + rules[ruleCount].v.intRange.start = b.template at(p); + rules[ruleCount].v.intRange.end = (uint32_t)(b.template at(p + 8) - rules[ruleCount].v.intRange.start); + rules[ruleCount].v.intRange.idx = b.template at(p + 16); + rules[ruleCount].v.intRange.format = (uint8_t)b[p + 18]; + break; } p += fieldLen; ++ruleCount; diff --git a/node/Network.cpp b/node/Network.cpp index 111e4736..a9e8539e 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -493,6 +493,35 @@ static _doZtFilterResult _doZtFilter( } } } break; + case ZT_NETWORK_RULE_MATCH_INTEGER_RANGE: { + uint64_t integer = 0; + const unsigned int bits = (rules[rn].v.intRange.format & 63) + 1; + const unsigned int bytes = ((bits + 8 - 1) / 8); // integer ceiling of division by 8 + if ((rules[rn].v.intRange.format & 0x80) == 0) { + // Big-endian + unsigned int idx = rules[rn].v.intRange.idx + (8 - bytes); + const unsigned int eof = idx + bytes; + if (eof <= frameLen) { + while (idx < eof) { + integer <<= 8; + integer |= frameData[idx++]; + } + } + integer &= 0xffffffffffffffffULL >> (64 - bits); + } else { + // Little-endian + unsigned int idx = rules[rn].v.intRange.idx; + const unsigned int eof = idx + bytes; + if (eof <= frameLen) { + while (idx < eof) { + integer >>= 8; + integer |= ((uint64_t)frameData[idx++]) << 56; + } + } + integer >>= (64 - bits); + } + thisRuleMatches = (uint8_t)((integer >= rules[rn].v.intRange.start)&&(integer <= (rules[rn].v.intRange.start + (uint64_t)rules[rn].v.intRange.end))); + } break; // The result of an unsupported MATCH is configurable at the network // level via a flag. -- cgit v1.2.3 From c7d370c17fc8b44f4d83868de9c6958347b75ca2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 24 Oct 2017 14:49:38 -0700 Subject: Delete something that turns out not to be useful. This will be handled differently. --- node/CertificateOfRepresentation.hpp | 188 ----------------------------------- node/Credential.hpp | 1 - node/IncomingPacket.cpp | 24 ----- node/Peer.cpp | 5 - node/Topology.cpp | 7 -- node/Topology.hpp | 21 ---- node/Trace.cpp | 13 --- node/Trace.hpp | 2 - 8 files changed, 261 deletions(-) delete mode 100644 node/CertificateOfRepresentation.hpp (limited to 'node') diff --git a/node/CertificateOfRepresentation.hpp b/node/CertificateOfRepresentation.hpp deleted file mode 100644 index 3007f1dc..00000000 --- a/node/CertificateOfRepresentation.hpp +++ /dev/null @@ -1,188 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CERTIFICATEOFREPRESENTATION_HPP -#define ZT_CERTIFICATEOFREPRESENTATION_HPP - -#include "Constants.hpp" -#include "Credential.hpp" -#include "Address.hpp" -#include "C25519.hpp" -#include "Identity.hpp" -#include "Buffer.hpp" - -/** - * Maximum number of addresses allowed in a COR - */ -#define ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES ZT_MAX_UPSTREAMS - -namespace ZeroTier { - -/** - * A signed enumeration of a node's roots (planet and moons) - * - * This is sent as part of HELLO and attests to which roots a node trusts - * to represent it on the network. Federated roots (moons) can send these - * further upstream to tell global roots which nodes they represent, making - * them reachable via federated roots if they are not reachable directly. - * - * As of 1.2.0 this is sent but not used. Right now nodes still always - * announce to planetary roots no matter what. In the future this can be - * used to implement even better fault tolerance for federation for the - * no roots are reachable case as well as a "privacy mode" where federated - * roots can shield nodes entirely and p2p connectivity behind them can - * be disabled. This will be desirable for a number of use cases. - */ -class CertificateOfRepresentation : public Credential -{ -public: - static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COR; } - - CertificateOfRepresentation() - { - memset(this,0,sizeof(CertificateOfRepresentation)); - } - - inline uint32_t id() const { return 0; } - inline uint64_t timestamp() const { return _timestamp; } - inline const Address &representative(const unsigned int i) const { return _reps[i]; } - inline unsigned int repCount() const { return _repCount; } - - inline void clear() - { - memset(this,0,sizeof(CertificateOfRepresentation)); - } - - /** - * Add a representative if space remains - * - * @param r Representative to add - * @return True if representative was added - */ - inline bool addRepresentative(const Address &r) - { - if (_repCount < ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES) { - _reps[_repCount++] = r; - return true; - } - return false; - } - - /** - * Sign this COR with my identity - * - * @param myIdentity This node's identity - * @param ts COR timestamp for establishing new vs. old - */ - inline void sign(const Identity &myIdentity,const uint64_t ts) - { - _timestamp = ts; - Buffer tmp; - this->serialize(tmp,true); - _signature = myIdentity.sign(tmp.data(),tmp.size()); - } - - /** - * Verify this COR's signature - * - * @param senderIdentity Identity of sender of COR - * @return True if COR is valid - */ - inline bool verify(const Identity &senderIdentity) - { - try { - Buffer tmp; - this->serialize(tmp,true); - return senderIdentity.verify(tmp.data(),tmp.size(),_signature.data,ZT_C25519_SIGNATURE_LEN); - } catch ( ... ) { - return false; - } - } - - template - inline void serialize(Buffer &b,const bool forSign = false) const - { - if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - - b.append((uint64_t)_timestamp); - b.append((uint16_t)_repCount); - for(unsigned int i=0;i<_repCount;++i) - _reps[i].appendTo(b); - - if (!forSign) { - b.append((uint8_t)1); // 1 == Ed25519 signature - b.append((uint16_t)ZT_C25519_SIGNATURE_LEN); - b.append(_signature.data,ZT_C25519_SIGNATURE_LEN); - } - - b.append((uint16_t)0); // size of any additional fields, currently 0 - - if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - } - - template - inline unsigned int deserialize(const Buffer &b,unsigned int startAt = 0) - { - clear(); - - unsigned int p = startAt; - - _timestamp = b.template at(p); p += 8; - const unsigned int rc = b.template at(p); p += 2; - for(unsigned int i=0;i ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES) ? ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES : rc; - - if (b[p++] == 1) { - if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { - p += 2; - memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); - p += ZT_C25519_SIGNATURE_LEN; - } else throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_CRYPTOGRAPHIC_TOKEN; - } else { - p += 2 + b.template at(p); - } - - p += 2 + b.template at(p); - if (p > b.size()) - throw ZT_EXCEPTION_INVALID_SERIALIZED_DATA_OVERFLOW; - - return (p - startAt); - } - -private: - uint64_t _timestamp; - Address _reps[ZT_CERTIFICATEOFREPRESENTATION_MAX_ADDRESSES]; - unsigned int _repCount; - C25519::Signature _signature; -}; - -} // namespace ZeroTier - -#endif diff --git a/node/Credential.hpp b/node/Credential.hpp index bc81919b..e8767e22 100644 --- a/node/Credential.hpp +++ b/node/Credential.hpp @@ -56,7 +56,6 @@ public: CREDENTIAL_TYPE_CAPABILITY = 2, CREDENTIAL_TYPE_TAG = 3, CREDENTIAL_TYPE_COO = 4, // CertificateOfOwnership - CREDENTIAL_TYPE_COR = 5, // CertificateOfRepresentation CREDENTIAL_TYPE_REVOCATION = 6 }; }; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index c0409c91..9b614e37 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -44,7 +44,6 @@ #include "World.hpp" #include "Node.hpp" #include "CertificateOfMembership.hpp" -#include "CertificateOfRepresentation.hpp" #include "Capability.hpp" #include "Tag.hpp" #include "Revocation.hpp" @@ -328,15 +327,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool ptr += 16; } } - - // Certificates of representation (if present) - if ((ptr + 2) <= size()) { - if (at(ptr) > 0) { - CertificateOfRepresentation cor; - ptr += 2; - ptr += cor.deserialize(*this,ptr); - } else ptr += 2; - } } // Send OK(HELLO) with an echo of the packet's timestamp and some of the same @@ -401,11 +391,6 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool } outp.setAt(worldUpdateSizeAt,(uint16_t)(outp.size() - (worldUpdateSizeAt + 2))); - const unsigned int corSizeAt = outp.size(); - outp.addSize(2); - RR->topology->appendCertificateOfRepresentation(outp); - outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); - outp.armor(peer->key(),true,_path->nextOutgoingCounter()); _path->send(RR,tPtr,outp.data(),outp.size(),now); @@ -460,15 +445,6 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP } } - // Handle certificate of representation if present - if ((ptr + 2) <= size()) { - if (at(ptr) > 0) { - CertificateOfRepresentation cor; - ptr += 2; - ptr += cor.deserialize(*this,ptr); - } else ptr += 2; - } - if (!hops()) peer->addDirectLatencyMeasurment((unsigned int)latency); peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); diff --git a/node/Peer.cpp b/node/Peer.cpp index 60661592..255d4004 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -343,11 +343,6 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA outp.append((uint64_t)0); } - const unsigned int corSizeAt = outp.size(); - outp.addSize(2); - RR->topology->appendCertificateOfRepresentation(outp); - outp.setAt(corSizeAt,(uint16_t)(outp.size() - (corSizeAt + 2))); - outp.cryptField(_key,startCryptedPortionAt,outp.size() - startCryptedPortionAt); RR->node->expectReplyTo(outp.packetId()); diff --git a/node/Topology.cpp b/node/Topology.cpp index f884e9c3..d5fea569 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -425,13 +425,6 @@ void Topology::_memoizeUpstreams(void *tPtr) } std::sort(_upstreamAddresses.begin(),_upstreamAddresses.end()); - - _cor.clear(); - for(std::vector
::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) { - if (!_cor.addRepresentative(*a)) - break; - } - _cor.sign(RR->identity,RR->node->now()); } void Topology::_savePeer(void *tPtr,const SharedPtr &peer) diff --git a/node/Topology.hpp b/node/Topology.hpp index c3a218e3..650e5363 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -46,7 +46,6 @@ #include "InetAddress.hpp" #include "Hashtable.hpp" #include "World.hpp" -#include "CertificateOfRepresentation.hpp" namespace ZeroTier { @@ -441,25 +440,6 @@ public: } } - /** - * @return Current certificate of representation (copy) - */ - inline CertificateOfRepresentation certificateOfRepresentation() const - { - Mutex::Lock _l(_upstreams_m); - return _cor; - } - - /** - * @param buf Buffer to receive COR - */ - template - void appendCertificateOfRepresentation(Buffer &buf) - { - Mutex::Lock _l(_upstreams_m); - _cor.serialize(buf); - } - private: Identity _getIdentity(void *tPtr,const Address &zta); void _memoizeUpstreams(void *tPtr); @@ -480,7 +460,6 @@ private: std::vector _moons; std::vector< std::pair > _moonSeeds; std::vector
_upstreamAddresses; - CertificateOfRepresentation _cor; bool _amRoot; Mutex _upstreams_m; // locks worlds, upstream info, moon info, etc. }; diff --git a/node/Trace.cpp b/node/Trace.cpp index 8e78b676..d90c3143 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -32,7 +32,6 @@ #include "Dictionary.hpp" #include "CertificateOfMembership.hpp" #include "CertificateOfOwnership.hpp" -#include "CertificateOfRepresentation.hpp" #include "Tag.hpp" #include "Capability.hpp" #include "Revocation.hpp" @@ -287,18 +286,6 @@ void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c, _send(tPtr,d,c.networkId()); } -void Trace::credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason) -{ - Dictionary d; - d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id()); - d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp()); - if (reason) - d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason); - _send(tPtr,d,0); -} - void Trace::credentialRejected(void *const tPtr,const Capability &c,const char *reason) { Dictionary d; diff --git a/node/Trace.hpp b/node/Trace.hpp index a7b2b194..5ee5b520 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -53,7 +53,6 @@ class NetworkConfig; class MAC; class CertificateOfMembership; class CertificateOfOwnership; -class CertificateOfRepresentation; class Revocation; class Tag; class Capability; @@ -137,7 +136,6 @@ public: void credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason); void credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason); - void credentialRejected(void *const tPtr,const CertificateOfRepresentation &c,const char *reason); void credentialRejected(void *const tPtr,const Capability &c,const char *reason); void credentialRejected(void *const tPtr,const Tag &c,const char *reason); void credentialRejected(void *const tPtr,const Revocation &c,const char *reason); -- cgit v1.2.3 From 2d0dc62a53dcbd46c3e2c796dfa655b805909f6b Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 24 Oct 2017 14:57:02 -0700 Subject: docs --- node/Packet.hpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'node') diff --git a/node/Packet.hpp b/node/Packet.hpp index db70e06f..cc055347 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -67,7 +67,6 @@ * + Multipart network configurations for large network configs * + Tags and Capabilities * + Inline push of CertificateOfMembership deprecated - * + Certificates of representation for federation and mesh * 9 - 1.2.0 ... CURRENT * + In-band encoding of packet counter for link quality measurement */ @@ -550,8 +549,6 @@ public: * [<[8] 64-bit world ID of moon>] * [<[8] 64-bit timestamp of moon>] * [... additional moon type/ID/timestamp tuples ...] - * <[2] 16-bit length of certificate of representation> - * [... certificate of representation ...] * * HELLO is sent in the clear as it is how peers share their identity * public keys. A few additional fields are sent in the clear too, but @@ -573,8 +570,6 @@ public: * <[...] physical destination address of packet> * <[2] 16-bit length of world update(s) or 0 if none> * [[...] updates to planets and/or moons] - * <[2] 16-bit length of certificate of representation> - * [... certificate of representation ...] * * With the exception of the timestamp, the other fields pertain to the * respondent who is sending OK and are not echoes. @@ -680,7 +675,7 @@ public: * 0x5 - REDIRECTed inbound frame * 0x6 - WATCHed inbound frame * 0x7 - (reserved for future use) - * + * * An extended frame carries full MAC addressing, making it a * superset of VERB_FRAME. It is used for bridged traffic, * redirected or observed traffic via rules, and can in theory @@ -762,7 +757,7 @@ public: * * It would be valid and correct as of 1.2.0 to use NETWORK_CONFIG always, * but OK(NTEWORK_CONFIG_REQUEST) should be sent for compatibility. - * + * * OK response payload: * <[8] 64-bit network ID> * <[2] 16-bit length of network configuration dictionary chunk> -- cgit v1.2.3 From 0d8b8d8426db50cc74b95be8d3878948954a75b3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 24 Oct 2017 15:04:19 -0700 Subject: Remove some unused constants. --- node/Constants.hpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 651fe50d..30cd1575 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -189,11 +189,6 @@ */ #define ZT_RX_QUEUE_SIZE 64 -/** - * RX queue entries older than this do not "exist" - */ -#define ZT_RX_QUEUE_EXPIRE 4000 - /** * Length of secret key in bytes -- 256-bit -- do not change */ @@ -209,11 +204,6 @@ */ #define ZT_HOUSEKEEPING_PERIOD 60000 -/** - * How long to remember peer records in RAM if they haven't been used - */ -#define ZT_PEER_IN_MEMORY_EXPIRATION 600000 - /** * Delay between WHOIS retries in ms */ @@ -232,7 +222,7 @@ /** * Maximum latency to allow for OK(HELLO) before packet is discarded */ -#define ZT_HELLO_MAX_ALLOWABLE_LATENCY 60000 +#define ZT_HELLO_MAX_ALLOWABLE_LATENCY 120000 /** * Maximum number of ZT hops allowed (this is not IP hops/TTL) @@ -241,11 +231,6 @@ */ #define ZT_RELAY_MAX_HOPS 3 -/** - * Maximum number of upstreams to use (far more than we should ever need) - */ -#define ZT_MAX_UPSTREAMS 64 - /** * Expire time for multicast 'likes' and indirect multicast memberships in ms */ -- cgit v1.2.3 From 459f1e7bfb50eb7b491940b7106d8788a7a5e11f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 25 Oct 2017 12:42:14 -0700 Subject: Refactor path stability stuff and add basic multipath support. --- node/Constants.hpp | 10 -- node/Hashtable.hpp | 5 - node/IncomingPacket.cpp | 7 +- node/InetAddress.cpp | 36 +++-- node/InetAddress.hpp | 4 +- node/Node.cpp | 24 ++-- node/Path.hpp | 30 ++++- node/Peer.cpp | 352 ++++++++++++++++++++++++++++++------------------ node/Peer.hpp | 173 ++++++++++-------------- node/Switch.cpp | 81 ++--------- node/Topology.cpp | 2 +- node/Topology.hpp | 2 +- node/Trace.cpp | 5 +- node/Trace.hpp | 2 +- 14 files changed, 375 insertions(+), 358 deletions(-) (limited to 'node') diff --git a/node/Constants.hpp b/node/Constants.hpp index 30cd1575..6360a693 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -268,16 +268,6 @@ */ #define ZT_PATH_HEARTBEAT_PERIOD 14000 -/** - * Paths are considered inactive if they have not received traffic in this long - */ -#define ZT_PATH_ALIVE_TIMEOUT 45000 - -/** - * Minimum time between attempts to check dead paths to see if they can be re-awakened - */ -#define ZT_PATH_MIN_REACTIVATE_INTERVAL 2500 - /** * Do not accept HELLOs over a given path more often than this */ diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp index 95a8e74f..e5496592 100644 --- a/node/Hashtable.hpp +++ b/node/Hashtable.hpp @@ -42,11 +42,6 @@ namespace ZeroTier { /** * A minimal hash table implementation for the ZeroTier core - * - * This is not a drop-in replacement for STL containers, and has several - * limitations. Keys can be uint64_t or an object, and if the latter they - * must implement a method called hashCode() that returns an unsigned long - * value that is evenly distributed. */ template class Hashtable diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 9b614e37..dfa0a161 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -446,7 +446,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP } if (!hops()) - peer->addDirectLatencyMeasurment((unsigned int)latency); + _path->updateLatency((unsigned int)latency); + peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision); if ((externalSurfaceAddress)&&(hops() == 0)) @@ -1091,7 +1092,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->redirect(tPtr,_path->localSocket(),a,now); + peer->clusterRedirect(tPtr,_path->localSocket(),a,now); } else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } @@ -1105,7 +1106,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->redirect(tPtr,_path->localSocket(),a,now); + peer->clusterRedirect(tPtr,_path->localSocket(),a,now); } else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } diff --git a/node/InetAddress.cpp b/node/InetAddress.cpp index f7585bdb..d3efc089 100644 --- a/node/InetAddress.cpp +++ b/node/InetAddress.cpp @@ -273,29 +273,27 @@ InetAddress InetAddress::network() const return r; } -#ifdef ZT_SDK - bool InetAddress::isEqualPrefix(const InetAddress &addr) const - { - if (addr.ss_family == ss_family) { - switch(ss_family) { - case AF_INET6: { - const InetAddress mask(netmask()); - InetAddress addr_mask(addr.netmask()); - const uint8_t *n = reinterpret_cast(reinterpret_cast(&addr_mask)->sin6_addr.s6_addr); - const uint8_t *m = reinterpret_cast(reinterpret_cast(&mask)->sin6_addr.s6_addr); - const uint8_t *a = reinterpret_cast(reinterpret_cast(&addr)->sin6_addr.s6_addr); - const uint8_t *b = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); - for(unsigned int i=0;i<16;++i) { - if ((a[i] & m[i]) != (b[i] & n[i])) - return false; - } - return true; +bool InetAddress::isEqualPrefix(const InetAddress &addr) const +{ + if (addr.ss_family == ss_family) { + switch(ss_family) { + case AF_INET6: { + const InetAddress mask(netmask()); + InetAddress addr_mask(addr.netmask()); + const uint8_t *n = reinterpret_cast(reinterpret_cast(&addr_mask)->sin6_addr.s6_addr); + const uint8_t *m = reinterpret_cast(reinterpret_cast(&mask)->sin6_addr.s6_addr); + const uint8_t *a = reinterpret_cast(reinterpret_cast(&addr)->sin6_addr.s6_addr); + const uint8_t *b = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); + for(unsigned int i=0;i<16;++i) { + if ((a[i] & m[i]) != (b[i] & n[i])) + return false; } + return true; } } - return false; } -#endif + return false; +} bool InetAddress::containsAddress(const InetAddress &addr) const { diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 61cdb05e..79bf76ad 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -330,7 +330,6 @@ struct InetAddress : public sockaddr_storage */ InetAddress network() const; -#ifdef ZT_SDK /** * Test whether this IPv6 prefix matches the prefix of a given IPv6 address * @@ -338,8 +337,7 @@ struct InetAddress : public sockaddr_storage * @return True if this IPv6 prefix matches the prefix of a given IPv6 address */ bool isEqualPrefix(const InetAddress &addr) const; -#endif - + /** * Test whether this IP/netmask contains this address * diff --git a/node/Node.cpp b/node/Node.cpp index 31ee8f19..b7dbffc3 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -191,12 +191,13 @@ public: { const std::vector *const upstreamStableEndpoints = _upstreamsToContact.get(p->address()); if (upstreamStableEndpoints) { - bool contacted = false; - // Upstreams must be pinged constantly over both IPv4 and IPv6 to allow // them to perform three way handshake introductions for both stacks. - if (!p->doPingAndKeepalive(_tPtr,_now,AF_INET)) { + const unsigned int sent = p->doPingAndKeepalive(_tPtr,_now); + bool contacted = (sent != 0); + + if ((sent & 0x1) == 0) { // bit 0x1 == IPv4 sent for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET) { @@ -205,8 +206,9 @@ public: break; } } - } else contacted = true; - if (!p->doPingAndKeepalive(_tPtr,_now,AF_INET6)) { + } + + if ((sent & 0x2) == 0) { // bit 0x2 == IPv6 sent for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) { const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()]; if (addr.ss_family == AF_INET6) { @@ -215,8 +217,10 @@ public: break; } } - } else contacted = true; + } + // If we have no memoized addresses for this upstream peer, attempt to contact + // it indirectly so we will be introduced. if ((!contacted)&&(_bestCurrentUpstream)) { const SharedPtr up(_bestCurrentUpstream->getBestPath(_now,true)); if (up) @@ -224,9 +228,11 @@ public: } lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream); - _upstreamsToContact.erase(p->address()); // erase from upstreams to contact so that we can WHOIS those that remain + + _upstreamsToContact.erase(p->address()); // after this we'll WHOIS all upstreams that remain } else if (p->isActive(_now)) { - p->doPingAndKeepalive(_tPtr,_now,-1); + // Regular non-upstream nodes get pinged if they appear active. + p->doPingAndKeepalive(_tPtr,_now); } } @@ -420,7 +426,7 @@ ZT_PeerList *Node::peers() const p->versionMinor = -1; p->versionRev = -1; } - p->latency = pi->second->latency(); + p->latency = pi->second->latency(_now); p->role = RR->topology->role(pi->second->identity().address()); std::vector< SharedPtr > paths(pi->second->paths(_now)); diff --git a/node/Path.hpp b/node/Path.hpp index 050fb6e2..80132c13 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -100,6 +100,7 @@ public: _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), + _latency(0xffff), _addr(), _ipScope(InetAddress::IP_SCOPE_NONE) { @@ -117,6 +118,7 @@ public: _incomingLinkQualitySlowLogCounter(-64), // discard first fast log _incomingLinkQualityPreviousPacketCounter(0), _outgoingPacketCounter(0), + _latency(0xffff), _addr(addr), _ipScope(addr.ipScope()) { @@ -188,6 +190,19 @@ public: */ inline void sent(const int64_t t) { _lastOut = t; } + /** + * Update path latency with a new measurement + * + * @param l Measured latency + */ + inline void updateLatency(const unsigned int l) + { + unsigned int pl = _latency; + if (pl < 0xffff) + _latency = (pl + l) / 2; + else _latency = l; + } + /** * @return Local socket as specified by external code */ @@ -259,9 +274,19 @@ public: } /** - * @return True if path appears alive + * @return Latency or 0xffff if unknown */ - inline bool alive(const int64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); } + inline unsigned int latency() const { return _latency; } + + /** + * @return Path quality -- lower is better + */ + inline int quality(const int64_t now) const + { + const int l = (int)_latency; + const int age = (int)std::min((now - _lastIn),(int64_t)(ZT_PATH_HEARTBEAT_PERIOD * 10)); // set an upper sanity limit to avoid overflow + return (((age < (ZT_PATH_HEARTBEAT_PERIOD + 5000)) ? l : (l + 0xffff + age)) * (int)((ZT_INETADDRESS_MAX_SCOPE - _ipScope) + 1)); + } /** * @return True if this path needs a heartbeat @@ -300,6 +325,7 @@ private: volatile signed int _incomingLinkQualitySlowLogCounter; volatile unsigned int _incomingLinkQualityPreviousPacketCounter; volatile unsigned int _outgoingPacketCounter; + volatile unsigned int _latency; InetAddress _addr; InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often volatile uint8_t _incomingLinkQualitySlowLog[32]; diff --git a/node/Peer.cpp b/node/Peer.cpp index 255d4004..61d8e990 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -52,12 +52,12 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastComRequestSent(0), _lastCredentialsReceived(0), _lastTrustEstablishedPacketReceived(0), + _lastSentFullHello(0), _vProto(0), _vMajor(0), _vMinor(0), _vRevision(0), _id(peerIdentity), - _latency(0), _directPathPushCutoffCount(0), _credentialsCutoffCount(0) { @@ -148,59 +148,47 @@ void Peer::received( if (hops == 0) { // If this is a direct packet (no hops), update existing paths or learn new ones - bool pathAlreadyKnown = false; - - { - Mutex::Lock _l(_paths_m); - if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) { - const struct sockaddr_in *const r = reinterpret_cast(&(path->address())); - const struct sockaddr_in *const l = reinterpret_cast(&(_v4Path.p->address())); - if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(path->localSocket() == _v4Path.p->localSocket())) { - _v4Path.lr = now; - pathAlreadyKnown = true; - } - } else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) { - const struct sockaddr_in6 *const r = reinterpret_cast(&(path->address())); - const struct sockaddr_in6 *const l = reinterpret_cast(&(_v6Path.p->address())); - if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(path->localSocket() == _v6Path.p->localSocket())) { - _v6Path.lr = now; - pathAlreadyKnown = true; - } - } - } - - if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address())) ) { - Mutex::Lock _l(_paths_m); + Mutex::Lock _l(_paths_m); - _PeerPath *replacablePath = (_PeerPath *)0; - if (path->address().ss_family == AF_INET) { - if ( ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || (path->preferenceRank() >= _v4Path.p->preferenceRank()) ) && ( (now - _v4Path.sticky) > ZT_PEER_PATH_EXPIRATION ) ) { - replacablePath = &_v4Path; + unsigned int worstQualityPath = 0; + int worstQuality = 0; + bool havePath = false; + for(unsigned int p=0;paddress().ss_family == AF_INET6) { - if ( ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || (path->preferenceRank() >= _v6Path.p->preferenceRank()) ) && ( (now - _v6Path.sticky) > ZT_PEER_PATH_EXPIRATION ) ) { - replacablePath = &_v6Path; + const int q = _paths[p].p->quality(now) / _paths[p].priority; + if (q >= worstQuality) { + worstQuality = q; + worstQualityPath = p; } + } else { + worstQualityPath = p; + break; } + } - if (replacablePath) { - if (verb == Packet::VERB_OK) { - RR->t->peerLearnedNewPath(tPtr,networkId,*this,replacablePath->p,path,packetId); - replacablePath->lr = now; - replacablePath->p = path; - } else { - RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); - attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); - path->sent(now); - } + if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) { + if (verb == Packet::VERB_OK) { + RR->t->peerLearnedNewPath(tPtr,networkId,*this,_paths[worstQualityPath].p,path,packetId); + _paths[worstQualityPath].lr = now; + _paths[worstQualityPath].p = path; + _paths[worstQualityPath].priority = 1; + } else { + attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); + path->sent(now); + RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); } } } - // If we are being relayed or if we're using a global address, send PUSH_DIRECT_PATHS. - // In the global address case we push only configured direct paths to accomplish - // fall-forward to local backplane networks over e.g. LAN or Amazon VPC. - if ( ((hops > 0)||(path->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) && (this->trustEstablished(now)) ) { + // If we have a trust relationship periodically push a message enumerating + // all known external addresses for ourselves. We now do this even if we + // have a current path since we'll want to use new ones too. + if (this->trustEstablished(now)) { if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) { _lastDirectPathPushSent = now; @@ -210,6 +198,7 @@ void Peer::received( for(std::vector::const_iterator i(dps.begin());i!=dps.end();++i) pathsToPush.push_back(*i); + // Do symmetric NAT prediction if we are communicating indirectly. if (hops > 0) { std::vector sym(RR->sa->getSymmetricNatPredictions()); for(unsigned long i=0,added=0;i Peer::getBestPath(int64_t now,bool includeExpired) const { Mutex::Lock _l(_paths_m); - int64_t v6lr = 0; - if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) - v6lr = _v6Path.p->lastIn(); - int64_t v4lr = 0; - if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) - v4lr = _v4Path.p->lastIn(); - - if ( (v6lr > v4lr) && ((now - v6lr) < ZT_PATH_ALIVE_TIMEOUT) ) { - return _v6Path.p->send(RR,tPtr,data,len,now); - } else if ((now - v4lr) < ZT_PATH_ALIVE_TIMEOUT) { - return _v4Path.p->send(RR,tPtr,data,len,now); - } else if (force) { - if (v6lr > v4lr) { - return _v6Path.p->send(RR,tPtr,data,len,now); - } else if (v4lr) { - return _v4Path.p->send(RR,tPtr,data,len,now); - } + unsigned int bestPath = ZT_PEER_MAX_PATHS; + int bestPathQuality = 2147483647; // INT_MAX + for(unsigned int i=0;iquality(now) / _paths[i].priority; + if (q < bestPathQuality) { + bestPathQuality = q; + bestPath = i; + } + } + } else break; } - return false; + if (bestPath != ZT_PEER_MAX_PATHS) + return _paths[bestPath].p; + return SharedPtr(); } -SharedPtr Peer::getBestPath(int64_t now,bool includeExpired) +void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &other) const { - Mutex::Lock _l(_paths_m); + unsigned int myBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + unsigned int myBestV6ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + int myBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + int myBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + unsigned int theirBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + unsigned int theirBestV6ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + int theirBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + int theirBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + for(int i=0;i<=ZT_INETADDRESS_MAX_SCOPE;++i) { + myBestV4ByScope[i] = ZT_PEER_MAX_PATHS; + myBestV6ByScope[i] = ZT_PEER_MAX_PATHS; + myBestV4QualityByScope[i] = 2147483647; + myBestV6QualityByScope[i] = 2147483647; + theirBestV4ByScope[i] = ZT_PEER_MAX_PATHS; + theirBestV6ByScope[i] = ZT_PEER_MAX_PATHS; + theirBestV4QualityByScope[i] = 2147483647; + theirBestV6QualityByScope[i] = 2147483647; + } - int64_t v6lr = 0; - if ((includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)) && (_v6Path.p)) { - v6lr = _v6Path.p->lastIn(); + Mutex::Lock _l1(_paths_m); + + for(unsigned int i=0;iquality(now) / _paths[i].priority; + const unsigned int s = (unsigned int)_paths[i].p->ipScope(); + switch(_paths[i].p->address().ss_family) { + case AF_INET: + if (q < myBestV4QualityByScope[s]) { + myBestV4QualityByScope[s] = q; + myBestV4ByScope[s] = i; + } + break; + case AF_INET6: + if (q < myBestV6QualityByScope[s]) { + myBestV6QualityByScope[s] = q; + myBestV6ByScope[s] = i; + } + break; + } + } else break; } - int64_t v4lr = 0; - if ((includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)) && (_v4Path.p)) { - v4lr = _v4Path.p->lastIn(); + + Mutex::Lock _l2(other->_paths_m); + + for(unsigned int i=0;i_paths[i].p) { + const int q = other->_paths[i].p->quality(now) / other->_paths[i].priority; + const unsigned int s = (unsigned int)other->_paths[i].p->ipScope(); + switch(other->_paths[i].p->address().ss_family) { + case AF_INET: + if (q < theirBestV4QualityByScope[s]) { + theirBestV4QualityByScope[s] = q; + theirBestV4ByScope[s] = i; + } + break; + case AF_INET6: + if (q < theirBestV6QualityByScope[s]) { + theirBestV6QualityByScope[s] = q; + theirBestV6ByScope[s] = i; + } + break; + } + } else break; } - if (v6lr > v4lr) { - return _v6Path.p; - } else if (v4lr) { - return _v4Path.p; + unsigned int mine = ZT_PEER_MAX_PATHS; + unsigned int theirs = ZT_PEER_MAX_PATHS; + + for(int s=ZT_INETADDRESS_MAX_SCOPE;s>=0;--s) { + if ((myBestV6ByScope[s] != ZT_PEER_MAX_PATHS)&&(theirBestV6ByScope[s] != ZT_PEER_MAX_PATHS)) { + mine = myBestV6ByScope[s]; + theirs = theirBestV6ByScope[s]; + break; + } + if ((myBestV4ByScope[s] != ZT_PEER_MAX_PATHS)&&(theirBestV4ByScope[s] != ZT_PEER_MAX_PATHS)) { + mine = myBestV4ByScope[s]; + theirs = theirBestV4ByScope[s]; + break; + } } - return SharedPtr(); + if (mine != ZT_PEER_MAX_PATHS) { + unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for black magickal NAT-t reasons + const unsigned int completed = alt + 2; + while (alt != completed) { + if ((alt & 1) == 0) { + Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((uint8_t)0); + other->_id.address().appendTo(outp); + outp.append((uint16_t)other->_paths[theirs].p->address().port()); + if (other->_paths[theirs].p->address().ss_family == AF_INET6) { + outp.append((uint8_t)16); + outp.append(other->_paths[theirs].p->address().rawIpData(),16); + } else { + outp.append((uint8_t)4); + outp.append(other->_paths[theirs].p->address().rawIpData(),4); + } + outp.armor(_key,true,_paths[mine].p->nextOutgoingCounter()); + _paths[mine].p->send(RR,tPtr,outp.data(),outp.size(),now); + } else { + Packet outp(other->_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS); + outp.append((uint8_t)0); + _id.address().appendTo(outp); + outp.append((uint16_t)_paths[mine].p->address().port()); + if (_paths[mine].p->address().ss_family == AF_INET6) { + outp.append((uint8_t)16); + outp.append(_paths[mine].p->address().rawIpData(),16); + } else { + outp.append((uint8_t)4); + outp.append(_paths[mine].p->address().rawIpData(),4); + } + outp.armor(other->_key,true,other->_paths[theirs].p->nextOutgoingCounter()); + other->_paths[theirs].p->send(RR,tPtr,outp.data(),outp.size(),now); + } + ++alt; + } + } } void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,unsigned int counter) @@ -377,76 +462,83 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now) } } -bool Peer::doPingAndKeepalive(void *tPtr,int64_t now,int inetAddressFamily) +unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) { + unsigned int sent = 0; + Mutex::Lock _l(_paths_m); - if (inetAddressFamily < 0) { - int64_t v6lr = 0; - if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) - v6lr = _v6Path.p->lastIn(); - int64_t v4lr = 0; - if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) - v4lr = _v4Path.p->lastIn(); - - if (v6lr > v4lr) { - if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v6Path.p->localSocket(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); - _v6Path.p->sent(now); - return true; - } - } else if (v4lr) { - if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v4Path.p->localSocket(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); - _v4Path.p->sent(now); - return true; - } - } - } else { - if ( (inetAddressFamily == AF_INET) && ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { - if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v4Path.p->localSocket(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); - _v4Path.p->sent(now); - return true; - } - } else if ( (inetAddressFamily == AF_INET6) && ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { - if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_v6Path.p->localSocket(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); - _v6Path.p->sent(now); - return true; + const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD); + _lastSentFullHello = now; + + unsigned int j = 0; + for(unsigned int i=0;ineedsHeartbeat(now))) { + attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello,_paths[i].p->nextOutgoingCounter()); + _paths[i].p->sent(now); + sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2; } + if (i != j) + _paths[j] = _paths[i]; + ++j; } } + while(j < ZT_PEER_MAX_PATHS) { + _paths[j].lr = 0; + _paths[j].p.zero(); + _paths[j].priority = 1; + ++j; + } - return false; + return sent; } -void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now) +void Peer::clusterRedirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now) { - if ((remoteAddress.ss_family != AF_INET)&&(remoteAddress.ss_family != AF_INET6)) // sanity check - return; - - SharedPtr op; SharedPtr np(RR->topology->getPath(localSocket,remoteAddress)); - np->received(now); + RR->t->peerRedirected(tPtr,0,*this,np); attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,np->nextOutgoingCounter()); - { Mutex::Lock _l(_paths_m); - if (remoteAddress.ss_family == AF_INET) { - op = _v4Path.p; - _v4Path.lr = now; - _v4Path.sticky = now; - _v4Path.p = np; - } else if (remoteAddress.ss_family == AF_INET6) { - op = _v6Path.p; - _v6Path.lr = now; - _v6Path.sticky = now; - _v6Path.p = np; + int worstQuality = 0; + unsigned int worstQualityPath = 0; + for(unsigned int i=0;iquality(now) / _paths[i].priority; + if (q >= worstQuality) { + worstQuality = q; + worstQualityPath = i; + } + } else { + worstQualityPath = i; + break; + } } + _paths[worstQualityPath].lr = now; + _paths[worstQualityPath].p = np; + _paths[worstQualityPath].priority = 6; // 1 + 5 } +} - RR->t->peerRedirected(tPtr,0,*this,op,np); +void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now) +{ + Mutex::Lock _l(_paths_m); + for(unsigned int i=0;iaddress().ss_family == inetAddressFamily)&&(_paths[i].p->ipScope() == scope)) { + attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,false,_paths[i].p->nextOutgoingCounter()); + _paths[i].p->sent(now); + _paths[i].lr = 0; // path will not be used unless it speaks again + } + } else break; + } } } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index e08f7d36..c236a2cd 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -53,6 +53,15 @@ #define ZT_PEER_MAX_SERIALIZED_STATE_SIZE (sizeof(Peer) + 32 + (sizeof(Path) * 2)) +/** + * Maximum number of direct paths to a peer + * + * This can be increased. You'll want about 2X the number of physical links + * you are ever likely to want to bundle/trunk since there is likely to be + * a path for every protocol (IPv4, IPv6, etc.). + */ +#define ZT_PEER_MAX_PATHS 16 + namespace ZeroTier { /** @@ -116,6 +125,8 @@ public: const uint64_t networkId); /** + * Check whether we have an active path to this peer via the given address + * * @param now Current time * @param addr Remote address * @return True if we have an active path to this destination @@ -123,7 +134,13 @@ public: inline bool hasActivePathTo(int64_t now,const InetAddress &addr) const { Mutex::Lock _l(_paths_m); - return ( ((addr.ss_family == AF_INET)&&(_v4Path.p)&&(_v4Path.p->address() == addr)&&(_v4Path.p->alive(now))) || ((addr.ss_family == AF_INET6)&&(_v6Path.p)&&(_v6Path.p->address() == addr)&&(_v6Path.p->alive(now))) ); + for(unsigned int i=0;iaddress() == addr)) + return true; + } else break; + } + return false; } /** @@ -136,19 +153,27 @@ public: * @param force If true, send even if path is not alive * @return True if we actually sent something */ - bool sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force); + inline bool sendDirect(void *tPtr,const void *data,unsigned int len,int64_t now,bool force) + { + SharedPtr bp(getBestPath(now,force)); + if (bp) + return bp->send(RR,tPtr,data,len,now); + return false; + } /** * Get the best current direct path * - * This does not check Path::alive(), but does return the most recently - * active path and does check expiration (which is a longer timeout). - * * @param now Current time * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none */ - SharedPtr getBestPath(int64_t now,bool includeExpired); + SharedPtr getBestPath(int64_t now,bool includeExpired) const; + + /** + * Send VERB_RENDEZVOUS to this and another peer via the best common IP scope and path + */ + void introduce(void *const tPtr,const int64_t now,const SharedPtr &other) const; /** * Send a HELLO to this peer at a specified physical address @@ -190,67 +215,39 @@ public: /** * Send pings or keepalives depending on configured timeouts * + * This also cleans up some internal data structures. It's called periodically from Node. + * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param now Current time * @param inetAddressFamily Keep this address family alive, or -1 for any - * @return True if we have at least one direct path of the given family (or any if family is -1) + * @return 0 if nothing sent or bit mask: bit 0x1 if IPv4 sent, bit 0x2 if IPv6 sent (0x3 means both sent) */ - bool doPingAndKeepalive(void *tPtr,int64_t now,int inetAddressFamily); + unsigned int doPingAndKeepalive(void *tPtr,int64_t now); /** - * Specify remote path for this peer and forget others - * - * This overrides normal path learning and tells this peer to be found - * at this address, at least within the address's family. Other address - * families are not modified. + * Process a cluster redirect sent by this peer * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param localSocket Local socket as supplied by external code * @param remoteAddress Remote address * @param now Current time */ - void redirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now); + void clusterRedirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now); /** * Reset paths within a given IP scope and address family * * Resetting a path involves sending an ECHO to it and then deactivating - * it until or unless it responds. + * it until or unless it responds. This is done when we detect a change + * to our external IP or another system change that might invalidate + * many or all current paths. * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call * @param scope IP scope * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now) - { - Mutex::Lock _l(_paths_m); - if ((inetAddressFamily == AF_INET)&&(_v4Path.lr)&&(_v4Path.p->address().ipScope() == scope)) { - attemptToContactAt(tPtr,_v4Path.p->localSocket(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); - _v4Path.p->sent(now); - _v4Path.lr = 0; // path will not be used unless it speaks again - } else if ((inetAddressFamily == AF_INET6)&&(_v6Path.lr)&&(_v6Path.p->address().ipScope() == scope)) { - attemptToContactAt(tPtr,_v6Path.p->localSocket(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); - _v6Path.p->sent(now); - _v6Path.lr = 0; // path will not be used unless it speaks again - } - } - - /** - * Fill parameters with V4 and V6 addresses if known and alive - * - * @param now Current time - * @param v4 Result parameter to receive active IPv4 address, if any - * @param v6 Result parameter to receive active IPv6 address, if any - */ - inline void getRendezvousAddresses(int64_t now,InetAddress &v4,InetAddress &v6) const - { - Mutex::Lock _l(_paths_m); - if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) - v4 = _v4Path.p->address(); - if (((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v6Path.p->alive(now))) - v6 = _v6Path.p->address(); - } + void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now); /** * @param now Current time @@ -260,10 +257,10 @@ public: { std::vector< SharedPtr > pp; Mutex::Lock _l(_paths_m); - if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) - pp.push_back(_v4Path.p); - if (((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v6Path.p->alive(now))) - pp.push_back(_v6Path.p); + for(unsigned int i=0;i bp(getBestPath(now,false)); + return ((bp) ? bp->latency() : 0xffff); + } /** * This computes a quality score for relays and root servers @@ -303,25 +304,12 @@ public: const uint64_t tsr = now - _lastReceive; if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT) return (~(unsigned int)0); - unsigned int l = _latency; + unsigned int l = latency(now); if (!l) l = 0xffff; return (l * (((unsigned int)tsr / (ZT_PEER_PING_PERIOD + 1000)) + 1)); } - /** - * Update latency with a new direct measurment - * - * @param l Direct latency measurment in ms - */ - inline void addDirectLatencyMeasurment(unsigned int l) - { - unsigned int ol = _latency; - if ((ol > 0)&&(ol < 10000)) - _latency = (ol + std::min(l,(unsigned int)65535)) / 2; - else _latency = std::min(l,(unsigned int)65535); - } - /** * @return 256-bit secret symmetric encryption key */ @@ -442,29 +430,15 @@ public: /** * Serialize a peer for storage in local cache * - * This does not serialize everything, just identity and addresses where the peer - * may be reached. + * This does not serialize everything, just non-ephemeral information. */ template - inline void serialize(Buffer &b) const + inline void serializeForCache(Buffer &b) const { - b.append((uint8_t)0); + b.append((uint8_t)1); _id.serialize(b); - b.append(_lastReceive); - b.append(_lastNontrivialReceive); - b.append(_lastTriedMemorizedPath); - b.append(_lastDirectPathPushSent); - b.append(_lastDirectPathPushReceive); - b.append(_lastCredentialRequestSent); - b.append(_lastWhoisRequestReceived); - b.append(_lastEchoRequestReceived); - b.append(_lastComRequestReceived); - b.append(_lastComRequestSent); - b.append(_lastCredentialsReceived); - b.append(_lastTrustEstablishedPacketReceived); - b.append((uint16_t)_vProto); b.append((uint16_t)_vMajor); b.append((uint16_t)_vMinor); @@ -472,15 +446,16 @@ public: { Mutex::Lock _l(_paths_m); - unsigned int pcount = 0; - if (_v4Path.p) ++pcount; - if (_v6Path.p) ++pcount; - b.append((uint8_t)pcount); - if (_v4Path.p) _v4Path.p->address().serialize(b); - if (_v6Path.p) _v6Path.p->address().serialize(b); + unsigned int pc = 0; + for(unsigned int i=0;iaddress().serialize(b); } - - b.append((uint16_t)0); } template @@ -488,7 +463,7 @@ public: { try { unsigned int ptr = 0; - if (b[ptr++] != 0) + if (b[ptr++] != 1) return SharedPtr(); Identity id; @@ -498,15 +473,16 @@ public: SharedPtr p(new Peer(renv,renv->identity,id)); - ptr += 12 * 8; // skip deserializing ephemeral state in this case - p->_vProto = b.template at(ptr); ptr += 2; p->_vMajor = b.template at(ptr); ptr += 2; p->_vMinor = b.template at(ptr); ptr += 2; p->_vRevision = b.template at(ptr); ptr += 2; - const unsigned int pcount = (unsigned int)b[ptr++]; - for(unsigned int i=0;i(ptr); ptr += 2; + for(unsigned int i=0;i p; + int priority; // >= 1, higher is better }; uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; @@ -548,19 +524,18 @@ private: int64_t _lastComRequestSent; int64_t _lastCredentialsReceived; int64_t _lastTrustEstablishedPacketReceived; + int64_t _lastSentFullHello; uint16_t _vProto; uint16_t _vMajor; uint16_t _vMinor; uint16_t _vRevision; - _PeerPath _v4Path; // IPv4 direct path - _PeerPath _v6Path; // IPv6 direct path + _PeerPath _paths[ZT_PEER_MAX_PATHS]; Mutex _paths_m; Identity _id; - unsigned int _latency; unsigned int _directPathPushCutoffCount; unsigned int _credentialsCutoffCount; diff --git a/node/Switch.cpp b/node/Switch.cpp index cc022b6b..a8cf0ce6 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -169,68 +169,22 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre if (packet.hops() < ZT_RELAY_MAX_HOPS) { packet.incrementHops(); - SharedPtr relayTo = RR->topology->getPeer(tPtr,destination); if ((relayTo)&&(relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,false))) { - if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { // don't send RENDEZVOUS for cluster frontplane relays - const InetAddress *hintToSource = (InetAddress *)0; - const InetAddress *hintToDest = (InetAddress *)0; - - InetAddress destV4,destV6; - InetAddress sourceV4,sourceV6; - relayTo->getRendezvousAddresses(now,destV4,destV6); - + if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) { const SharedPtr sourcePeer(RR->topology->getPeer(tPtr,source)); - if (sourcePeer) { - sourcePeer->getRendezvousAddresses(now,sourceV4,sourceV6); - if ((destV6)&&(sourceV6)) { - hintToSource = &destV6; - hintToDest = &sourceV6; - } else if ((destV4)&&(sourceV4)) { - hintToSource = &destV4; - hintToDest = &sourceV4; - } - - if ((hintToSource)&&(hintToDest)) { - unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for obscure NAT-t reasons - const unsigned int completed = alt + 2; - while (alt != completed) { - if ((alt & 1) == 0) { - Packet outp(source,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((uint8_t)0); - destination.appendTo(outp); - outp.append((uint16_t)hintToSource->port()); - if (hintToSource->ss_family == AF_INET6) { - outp.append((uint8_t)16); - outp.append(hintToSource->rawIpData(),16); - } else { - outp.append((uint8_t)4); - outp.append(hintToSource->rawIpData(),4); - } - send(tPtr,outp,true); - } else { - Packet outp(destination,RR->identity.address(),Packet::VERB_RENDEZVOUS); - outp.append((uint8_t)0); - source.appendTo(outp); - outp.append((uint16_t)hintToDest->port()); - if (hintToDest->ss_family == AF_INET6) { - outp.append((uint8_t)16); - outp.append(hintToDest->rawIpData(),16); - } else { - outp.append((uint8_t)4); - outp.append(hintToDest->rawIpData(),4); - } - send(tPtr,outp,true); - } - ++alt; - } - } - } + if (sourcePeer) + relayTo->introduce(tPtr,now,sourcePeer); } } else { relayTo = RR->topology->getUpstreamPeer(); - if ((relayTo)&&(relayTo->address() != source)) - relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true); + if ((relayTo)&&(relayTo->address() != source)) { + if (relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true)) { + const SharedPtr sourcePeer(RR->topology->getPeer(tPtr,source)); + if (sourcePeer) + relayTo->introduce(tPtr,now,sourcePeer); + } + } } } } else if ((reinterpret_cast(data)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED) != 0) { @@ -694,22 +648,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt) const SharedPtr peer(RR->topology->getPeer(tPtr,destination)); if (peer) { - /* First get the best path, and if it's dead (and this is not a root) - * we attempt to re-activate that path but this packet will flow - * upstream. If the path comes back alive, it will be used in the future. - * For roots we don't do the alive check since roots are not required - * to send heartbeats "down" and because we have to at least try to - * go somewhere. */ - viaPath = peer->getBestPath(now,false); - if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) { - if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(int64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) { - peer->attemptToContactAt(tPtr,viaPath->localSocket(),viaPath->address(),now,false,viaPath->nextOutgoingCounter()); - viaPath->sent(now); - } - viaPath.zero(); - } - if (!viaPath) { peer->tryMemorizedPath(tPtr,now); // periodically attempt memorized or statically defined paths, if any are known const SharedPtr relay(RR->topology->getUpstreamPeer()); diff --git a/node/Topology.cpp b/node/Topology.cpp index d5fea569..d1b389df 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -431,7 +431,7 @@ void Topology::_savePeer(void *tPtr,const SharedPtr &peer) { try { Buffer buf; - peer->serialize(buf); + peer->serializeForCache(buf); uint64_t tmpid[2]; tmpid[0] = peer->address().toInt(); tmpid[1] = 0; RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER,tmpid,buf.data(),buf.size()); } catch ( ... ) {} // sanity check, discard invalid entries diff --git a/node/Topology.hpp b/node/Topology.hpp index 650e5363..b09f95cf 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -300,7 +300,7 @@ public: SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { const SharedPtr pp((*p)->getBestPath(now,false)); - if ((pp)&&(pp->alive(now))) + if (pp) ++cnt; } return cnt; diff --git a/node/Trace.cpp b/node/Trace.cpp index d90c3143..6d85942d 100644 --- a/node/Trace.cpp +++ b/node/Trace.cpp @@ -92,16 +92,13 @@ void Trace::peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &p _send(tPtr,d,networkId); } -void Trace::peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath) +void Trace::peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &newPath) { char tmp[128]; Dictionary d; d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S); d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId); d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address()); - if (oldPath) { - d.add(ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR,oldPath->address().toString(tmp)); - } if (newPath) { d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,newPath->address().toString(tmp)); d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,newPath->localSocket()); diff --git a/node/Trace.hpp b/node/Trace.hpp index 5ee5b520..4192d1c2 100644 --- a/node/Trace.hpp +++ b/node/Trace.hpp @@ -105,7 +105,7 @@ public: void peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &path,const uint64_t packetId,const Packet::Verb verb); void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath,const uint64_t packetId); - void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &oldPath,const SharedPtr &newPath); + void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr &newPath); void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason); void incomingPacketInvalid(void *const tPtr,const SharedPtr &path,const uint64_t packetId,const Address &source,const unsigned int hops,const Packet::Verb verb,const char *reason); -- cgit v1.2.3 From 71bdaa95087536954f1f1cb7b4652fd9b33be587 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 25 Oct 2017 13:27:28 -0700 Subject: Now with more worky. --- include/ZeroTierOne.h | 6 +++--- node/Node.cpp | 2 ++ node/Peer.cpp | 48 ++++++++++++++++++++++++------------------------ node/Peer.hpp | 21 +++++++-------------- 4 files changed, 36 insertions(+), 41 deletions(-) (limited to 'node') diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 02bb283b..16668534 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -173,7 +173,7 @@ extern "C" { /** * Maximum number of direct network paths to a given peer */ -#define ZT_MAX_PEER_NETWORK_PATHS 4 +#define ZT_MAX_PEER_NETWORK_PATHS 16 /** * Maximum number of path configurations that can be set @@ -1228,9 +1228,9 @@ typedef struct int versionRev; /** - * Last measured latency in milliseconds or zero if unknown + * Last measured latency in milliseconds or -1 if unknown */ - unsigned int latency; + int latency; /** * What trust hierarchy role does this device have? diff --git a/node/Node.cpp b/node/Node.cpp index b7dbffc3..f0fcb4d7 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -427,6 +427,8 @@ ZT_PeerList *Node::peers() const p->versionRev = -1; } p->latency = pi->second->latency(_now); + if (p->latency >= 0xffff) + p->latency = -1; p->role = RR->topology->role(pi->second->identity().address()); std::vector< SharedPtr > paths(pi->second->paths(_now)); diff --git a/node/Peer.cpp b/node/Peer.cpp index 61d8e990..d2692011 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -153,7 +153,7 @@ void Peer::received( unsigned int worstQualityPath = 0; int worstQuality = 0; bool havePath = false; - for(unsigned int p=0;p Peer::getBestPath(int64_t now,bool includeExpired) const { Mutex::Lock _l(_paths_m); - unsigned int bestPath = ZT_PEER_MAX_PATHS; + unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS; int bestPathQuality = 2147483647; // INT_MAX - for(unsigned int i=0;iquality(now) / _paths[i].priority; - if (q < bestPathQuality) { + if (q <= bestPathQuality) { bestPathQuality = q; bestPath = i; } @@ -271,7 +271,7 @@ SharedPtr Peer::getBestPath(int64_t now,bool includeExpired) const } else break; } - if (bestPath != ZT_PEER_MAX_PATHS) + if (bestPath != ZT_MAX_PEER_NETWORK_PATHS) return _paths[bestPath].p; return SharedPtr(); } @@ -287,31 +287,31 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o int theirBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; int theirBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; for(int i=0;i<=ZT_INETADDRESS_MAX_SCOPE;++i) { - myBestV4ByScope[i] = ZT_PEER_MAX_PATHS; - myBestV6ByScope[i] = ZT_PEER_MAX_PATHS; + myBestV4ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS; + myBestV6ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS; myBestV4QualityByScope[i] = 2147483647; myBestV6QualityByScope[i] = 2147483647; - theirBestV4ByScope[i] = ZT_PEER_MAX_PATHS; - theirBestV6ByScope[i] = ZT_PEER_MAX_PATHS; + theirBestV4ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS; + theirBestV6ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS; theirBestV4QualityByScope[i] = 2147483647; theirBestV6QualityByScope[i] = 2147483647; } Mutex::Lock _l1(_paths_m); - for(unsigned int i=0;iquality(now) / _paths[i].priority; const unsigned int s = (unsigned int)_paths[i].p->ipScope(); switch(_paths[i].p->address().ss_family) { case AF_INET: - if (q < myBestV4QualityByScope[s]) { + if (q <= myBestV4QualityByScope[s]) { myBestV4QualityByScope[s] = q; myBestV4ByScope[s] = i; } break; case AF_INET6: - if (q < myBestV6QualityByScope[s]) { + if (q <= myBestV6QualityByScope[s]) { myBestV6QualityByScope[s] = q; myBestV6ByScope[s] = i; } @@ -322,19 +322,19 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o Mutex::Lock _l2(other->_paths_m); - for(unsigned int i=0;i_paths[i].p) { const int q = other->_paths[i].p->quality(now) / other->_paths[i].priority; const unsigned int s = (unsigned int)other->_paths[i].p->ipScope(); switch(other->_paths[i].p->address().ss_family) { case AF_INET: - if (q < theirBestV4QualityByScope[s]) { + if (q <= theirBestV4QualityByScope[s]) { theirBestV4QualityByScope[s] = q; theirBestV4ByScope[s] = i; } break; case AF_INET6: - if (q < theirBestV6QualityByScope[s]) { + if (q <= theirBestV6QualityByScope[s]) { theirBestV6QualityByScope[s] = q; theirBestV6ByScope[s] = i; } @@ -343,23 +343,23 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o } else break; } - unsigned int mine = ZT_PEER_MAX_PATHS; - unsigned int theirs = ZT_PEER_MAX_PATHS; + unsigned int mine = ZT_MAX_PEER_NETWORK_PATHS; + unsigned int theirs = ZT_MAX_PEER_NETWORK_PATHS; for(int s=ZT_INETADDRESS_MAX_SCOPE;s>=0;--s) { - if ((myBestV6ByScope[s] != ZT_PEER_MAX_PATHS)&&(theirBestV6ByScope[s] != ZT_PEER_MAX_PATHS)) { + if ((myBestV6ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)&&(theirBestV6ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)) { mine = myBestV6ByScope[s]; theirs = theirBestV6ByScope[s]; break; } - if ((myBestV4ByScope[s] != ZT_PEER_MAX_PATHS)&&(theirBestV4ByScope[s] != ZT_PEER_MAX_PATHS)) { + if ((myBestV4ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)&&(theirBestV4ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)) { mine = myBestV4ByScope[s]; theirs = theirBestV4ByScope[s]; break; } } - if (mine != ZT_PEER_MAX_PATHS) { + if (mine != ZT_MAX_PEER_NETWORK_PATHS) { unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for black magickal NAT-t reasons const unsigned int completed = alt + 2; while (alt != completed) { @@ -472,7 +472,7 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) _lastSentFullHello = now; unsigned int j = 0; - for(unsigned int i=0;ineedsHeartbeat(now))) { @@ -485,7 +485,7 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) ++j; } } - while(j < ZT_PEER_MAX_PATHS) { + while(j < ZT_MAX_PEER_NETWORK_PATHS) { _paths[j].lr = 0; _paths[j].p.zero(); _paths[j].priority = 1; @@ -504,7 +504,7 @@ void Peer::clusterRedirect(void *tPtr,const int64_t localSocket,const InetAddres Mutex::Lock _l(_paths_m); int worstQuality = 0; unsigned int worstQualityPath = 0; - for(unsigned int i=0;iaddress().ss_family == inetAddressFamily)&&(_paths[i].p->ipScope() == scope)) { attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,false,_paths[i].p->nextOutgoingCounter()); diff --git a/node/Peer.hpp b/node/Peer.hpp index c236a2cd..997c44f5 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -53,15 +53,6 @@ #define ZT_PEER_MAX_SERIALIZED_STATE_SIZE (sizeof(Peer) + 32 + (sizeof(Path) * 2)) -/** - * Maximum number of direct paths to a peer - * - * This can be increased. You'll want about 2X the number of physical links - * you are ever likely to want to bundle/trunk since there is likely to be - * a path for every protocol (IPv4, IPv6, etc.). - */ -#define ZT_PEER_MAX_PATHS 16 - namespace ZeroTier { /** @@ -134,7 +125,7 @@ public: inline bool hasActivePathTo(int64_t now,const InetAddress &addr) const { Mutex::Lock _l(_paths_m); - for(unsigned int i=0;iaddress() == addr)) return true; @@ -257,7 +248,7 @@ public: { std::vector< SharedPtr > pp; Mutex::Lock _l(_paths_m); - for(unsigned int i=0;i bp(getBestPath(now,false)); - return ((bp) ? bp->latency() : 0xffff); + if (bp) + return bp->latency(); + return 0xffff; } /** @@ -447,7 +440,7 @@ public: { Mutex::Lock _l(_paths_m); unsigned int pc = 0; - for(unsigned int i=0;i Date: Wed, 25 Oct 2017 15:44:10 -0700 Subject: A few fixes for cluster mode. --- node/IncomingPacket.cpp | 4 +- node/Path.hpp | 13 ++-- node/Peer.cpp | 183 +++++++++++++++++++++++++++++++----------------- node/Peer.hpp | 6 +- 4 files changed, 133 insertions(+), 73 deletions(-) (limited to 'node') diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index dfa0a161..d44e3b54 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -1092,7 +1092,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->clusterRedirect(tPtr,_path->localSocket(),a,now); + peer->clusterRedirect(tPtr,_path,a,now); } else if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } @@ -1106,7 +1106,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localSocket(),a)) ) // should use path { if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->clusterRedirect(tPtr,_path->localSocket(),a,now); + peer->clusterRedirect(tPtr,_path,a,now); } else if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); } diff --git a/node/Path.hpp b/node/Path.hpp index 80132c13..ab52ced6 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -281,13 +281,18 @@ public: /** * @return Path quality -- lower is better */ - inline int quality(const int64_t now) const + inline long quality(const int64_t now) const { - const int l = (int)_latency; - const int age = (int)std::min((now - _lastIn),(int64_t)(ZT_PATH_HEARTBEAT_PERIOD * 10)); // set an upper sanity limit to avoid overflow - return (((age < (ZT_PATH_HEARTBEAT_PERIOD + 5000)) ? l : (l + 0xffff + age)) * (int)((ZT_INETADDRESS_MAX_SCOPE - _ipScope) + 1)); + const int l = (long)_latency; + const int age = (long)std::min((now - _lastIn),(int64_t)(ZT_PATH_HEARTBEAT_PERIOD * 10)); // set an upper sanity limit to avoid overflow + return (((age < (ZT_PATH_HEARTBEAT_PERIOD + 5000)) ? l : (l + 0xffff + age)) * (long)((ZT_INETADDRESS_MAX_SCOPE - _ipScope) + 1)); } + /** + * @return True if this path is alive (receiving heartbeats) + */ + inline bool alive(const int64_t now) const { return ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000)); } + /** * @return True if this path needs a heartbeat */ diff --git a/node/Peer.cpp b/node/Peer.cpp index d2692011..d68e0df3 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -148,39 +148,64 @@ void Peer::received( if (hops == 0) { // If this is a direct packet (no hops), update existing paths or learn new ones - Mutex::Lock _l(_paths_m); - unsigned int worstQualityPath = 0; - int worstQuality = 0; bool havePath = false; - for(unsigned int p=0;pquality(now) / _paths[p].priority; - if (q >= worstQuality) { - worstQuality = q; - worstQualityPath = p; - } - } else { - worstQualityPath = p; - break; + { + Mutex::Lock _l(_paths_m); + for(unsigned int i=0;inode->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) { - if (verb == Packet::VERB_OK) { - RR->t->peerLearnedNewPath(tPtr,networkId,*this,_paths[worstQualityPath].p,path,packetId); - _paths[worstQualityPath].lr = now; - _paths[worstQualityPath].p = path; - _paths[worstQualityPath].priority = 1; - } else { - attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); - path->sent(now); - RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); + Mutex::Lock _l(_paths_m); + + // Paths are redunant if they duplicate an alive path to the same IP or + // with the same local socket and address family. + bool redundant = false; + for(unsigned int i=0;ialive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual(path->address())) ) ) { + redundant = true; + break; + } + } else break; + } + + if (!redundant) { + unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS; + int replacePathQuality = 0; + for(unsigned int i=0;iquality(now); + if (q > replacePathQuality) { + replacePathQuality = q; + replacePath = i; + } + } else { + replacePath = i; + break; + } + } + + if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) { + if (verb == Packet::VERB_OK) { + RR->t->peerLearnedNewPath(tPtr,networkId,*this,_paths[replacePath].p,path,packetId); + _paths[replacePath].lr = now; + _paths[replacePath].p = path; + _paths[replacePath].priority = 1; + } else { + attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); + path->sent(now); + RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb); + } + } } } } @@ -258,11 +283,11 @@ SharedPtr Peer::getBestPath(int64_t now,bool includeExpired) const Mutex::Lock _l(_paths_m); unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS; - int bestPathQuality = 2147483647; // INT_MAX + long bestPathQuality = 2147483647; for(unsigned int i=0;iquality(now) / _paths[i].priority; + const long q = _paths[i].p->quality(now) / _paths[i].priority; if (q <= bestPathQuality) { bestPathQuality = q; bestPath = i; @@ -280,12 +305,12 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o { unsigned int myBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; unsigned int myBestV6ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; - int myBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; - int myBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + long myBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + long myBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; unsigned int theirBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; unsigned int theirBestV6ByScope[ZT_INETADDRESS_MAX_SCOPE+1]; - int theirBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; - int theirBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + long theirBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; + long theirBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1]; for(int i=0;i<=ZT_INETADDRESS_MAX_SCOPE;++i) { myBestV4ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS; myBestV6ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS; @@ -301,7 +326,7 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o for(unsigned int i=0;iquality(now) / _paths[i].priority; + const long q = _paths[i].p->quality(now) / _paths[i].priority; const unsigned int s = (unsigned int)_paths[i].p->ipScope(); switch(_paths[i].p->address().ss_family) { case AF_INET: @@ -324,7 +349,7 @@ void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr &o for(unsigned int i=0;i_paths[i].p) { - const int q = other->_paths[i].p->quality(now) / other->_paths[i].priority; + const long q = other->_paths[i].p->quality(now) / other->_paths[i].priority; const unsigned int s = (unsigned int)other->_paths[i].p->ipScope(); switch(other->_paths[i].p->address().ss_family) { case AF_INET: @@ -471,19 +496,32 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD); _lastSentFullHello = now; + // Right now we only keep pinging links that have the maximum priority. The + // priority is used to track cluster redirections, meaning that when a cluster + // redirects us its redirect target links override all other links and we + // let those old links expire. + long maxPriority = 0; + for(unsigned int i=0;ineedsHeartbeat(now))) { - attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello,_paths[i].p->nextOutgoingCounter()); - _paths[i].p->sent(now); - sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2; + if (_paths[i].p) { + // Clean expired and reduced priority paths + if ( ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) && (_paths[i].priority == maxPriority) ) { + if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))) { + attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello,_paths[i].p->nextOutgoingCounter()); + _paths[i].p->sent(now); + sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2; + } + if (i != j) + _paths[j] = _paths[i]; + ++j; } - if (i != j) - _paths[j] = _paths[i]; - ++j; - } + } else break; } while(j < ZT_MAX_PEER_NETWORK_PATHS) { _paths[j].lr = 0; @@ -495,35 +533,52 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now) return sent; } -void Peer::clusterRedirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now) +void Peer::clusterRedirect(void *tPtr,const SharedPtr &originatingPath,const InetAddress &remoteAddress,const int64_t now) { - SharedPtr np(RR->topology->getPath(localSocket,remoteAddress)); + SharedPtr np(RR->topology->getPath(originatingPath->localSocket(),remoteAddress)); RR->t->peerRedirected(tPtr,0,*this,np); - attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,np->nextOutgoingCounter()); + + attemptToContactAt(tPtr,originatingPath->localSocket(),remoteAddress,now,true,np->nextOutgoingCounter()); + { Mutex::Lock _l(_paths_m); - int worstQuality = 0; - unsigned int worstQualityPath = 0; + + // New priority is higher than the priority of the originating path (if known) + long newPriority = 1; for(unsigned int i=0;iquality(now) / _paths[i].priority; - if (q >= worstQuality) { - worstQuality = q; - worstQualityPath = i; + } else break; + } + newPriority += 2; + + // Erase any paths with lower priority than this one or that are duplicate + // IPs and add this path. + unsigned int j = 0; + for(unsigned int i=0;i= newPriority)&&(!_paths[i].p->address().ipsEqual(remoteAddress))) { + if (i != j) + _paths[j] = _paths[i]; + ++j; } - } else { - worstQualityPath = i; - break; } } - _paths[worstQualityPath].lr = now; - _paths[worstQualityPath].p = np; - _paths[worstQualityPath].priority = 6; // 1 + 5 + if (j < ZT_MAX_PEER_NETWORK_PATHS) { + _paths[j].lr = now; + _paths[j].p = np; + _paths[j].priority = newPriority; + ++j; + while (j < ZT_MAX_PEER_NETWORK_PATHS) { + _paths[j].lr = 0; + _paths[j].p.zero(); + _paths[j].priority = 1; + ++j; + } + } } } diff --git a/node/Peer.hpp b/node/Peer.hpp index 997c44f5..53b916ab 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -219,11 +219,11 @@ public: * Process a cluster redirect sent by this peer * * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param localSocket Local socket as supplied by external code + * @param originatingPath Path from which redirect originated * @param remoteAddress Remote address * @param now Current time */ - void clusterRedirect(void *tPtr,const int64_t localSocket,const InetAddress &remoteAddress,const int64_t now); + void clusterRedirect(void *tPtr,const SharedPtr &originatingPath,const InetAddress &remoteAddress,const int64_t now); /** * Reset paths within a given IP scope and address family @@ -498,7 +498,7 @@ private: _PeerPath() : lr(0),p(),priority(1) {} int64_t lr; // time of last valid ZeroTier packet SharedPtr p; - int priority; // >= 1, higher is better + long priority; // >= 1, higher is better }; uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; -- cgit v1.2.3 From fac7dc9c913a94550692c31ca6c24fa4db5b5b52 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 25 Oct 2017 16:01:36 -0700 Subject: Stop duplciate IPv6 addresses due to privacy mode IPs. --- node/InetAddress.hpp | 20 ++++++++++++++++++++ node/Peer.cpp | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) (limited to 'node') diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 79bf76ad..c1ea6c13 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -405,6 +405,26 @@ struct InetAddress : public sockaddr_storage return false; } + /** + * Performs an IP-only comparison or, if that is impossible, a memcmp() + * + * This version compares only the first 64 bits of IPv6 addresses. + * + * @param a InetAddress to compare again + * @return True if only IP portions are equal (false for non-IP or null addresses) + */ + inline bool ipsEqual2(const InetAddress &a) const + { + if (ss_family == a.ss_family) { + if (ss_family == AF_INET) + return (reinterpret_cast(this)->sin_addr.s_addr == reinterpret_cast(&a)->sin_addr.s_addr); + if (ss_family == AF_INET6) + return (memcmp(reinterpret_cast(this)->sin6_addr.s6_addr,reinterpret_cast(&a)->sin6_addr.s6_addr,8) == 0); + return (memcmp(this,&a,sizeof(InetAddress)) == 0); + } + return false; + } + inline unsigned long hashCode() const { if (ss_family == AF_INET) { diff --git a/node/Peer.cpp b/node/Peer.cpp index d68e0df3..a3682a97 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -171,7 +171,7 @@ void Peer::received( bool redundant = false; for(unsigned int i=0;ialive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual(path->address())) ) ) { + if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) { redundant = true; break; } @@ -560,7 +560,7 @@ void Peer::clusterRedirect(void *tPtr,const SharedPtr &originatingPath,con unsigned int j = 0; for(unsigned int i=0;i= newPriority)&&(!_paths[i].p->address().ipsEqual(remoteAddress))) { + if ((_paths[i].priority >= newPriority)&&(!_paths[i].p->address().ipsEqual2(remoteAddress))) { if (i != j) _paths[j] = _paths[i]; ++j; -- cgit v1.2.3 From 4e88c80a22b6ca982341413ee806ade0df57b4b7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 2 Nov 2017 07:05:11 -0700 Subject: RethinkDB native connector work, minor fixes. --- .gitignore | 4 + controller/RethinkDB.cpp | 308 +++ controller/RethinkDB.hpp | 101 + ext/librethinkdbxx/.travis.yml | 11 + ext/librethinkdbxx/COPYRIGHT | 16 + ext/librethinkdbxx/Makefile | 126 + ext/librethinkdbxx/README.md | 72 + ext/librethinkdbxx/reql/add_docs.py | 80 + ext/librethinkdbxx/reql/gen.py | 33 + ext/librethinkdbxx/reql/python_docs.txt | 189 ++ ext/librethinkdbxx/reql/ql2.proto | 843 +++++++ ext/librethinkdbxx/src/connection.cc | 431 ++++ ext/librethinkdbxx/src/connection.h | 59 + ext/librethinkdbxx/src/connection_p.h | 133 + ext/librethinkdbxx/src/cursor.cc | 221 ++ ext/librethinkdbxx/src/cursor.h | 76 + ext/librethinkdbxx/src/cursor_p.h | 29 + ext/librethinkdbxx/src/datum.cc | 449 ++++ ext/librethinkdbxx/src/datum.h | 287 +++ ext/librethinkdbxx/src/error.h | 46 + ext/librethinkdbxx/src/exceptions.h | 13 + ext/librethinkdbxx/src/json.cc | 62 + ext/librethinkdbxx/src/json_p.h | 19 + ext/librethinkdbxx/src/rapidjson-config.h | 8 + ext/librethinkdbxx/src/rapidjson/allocators.h | 263 ++ ext/librethinkdbxx/src/rapidjson/document.h | 2575 ++++++++++++++++++++ ext/librethinkdbxx/src/rapidjson/encodedstream.h | 299 +++ ext/librethinkdbxx/src/rapidjson/encodings.h | 716 ++++++ ext/librethinkdbxx/src/rapidjson/error/en.h | 74 + ext/librethinkdbxx/src/rapidjson/error/error.h | 155 ++ ext/librethinkdbxx/src/rapidjson/filereadstream.h | 99 + ext/librethinkdbxx/src/rapidjson/filewritestream.h | 104 + ext/librethinkdbxx/src/rapidjson/fwd.h | 151 ++ .../src/rapidjson/internal/biginteger.h | 290 +++ ext/librethinkdbxx/src/rapidjson/internal/diyfp.h | 258 ++ ext/librethinkdbxx/src/rapidjson/internal/dtoa.h | 245 ++ .../src/rapidjson/internal/ieee754.h | 78 + ext/librethinkdbxx/src/rapidjson/internal/itoa.h | 304 +++ ext/librethinkdbxx/src/rapidjson/internal/meta.h | 181 ++ ext/librethinkdbxx/src/rapidjson/internal/pow10.h | 55 + ext/librethinkdbxx/src/rapidjson/internal/regex.h | 701 ++++++ ext/librethinkdbxx/src/rapidjson/internal/stack.h | 230 ++ .../src/rapidjson/internal/strfunc.h | 55 + ext/librethinkdbxx/src/rapidjson/internal/strtod.h | 269 ++ ext/librethinkdbxx/src/rapidjson/internal/swap.h | 46 + ext/librethinkdbxx/src/rapidjson/istreamwrapper.h | 115 + ext/librethinkdbxx/src/rapidjson/memorybuffer.h | 70 + ext/librethinkdbxx/src/rapidjson/memorystream.h | 71 + .../src/rapidjson/msinttypes/inttypes.h | 316 +++ .../src/rapidjson/msinttypes/stdint.h | 300 +++ ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h | 81 + ext/librethinkdbxx/src/rapidjson/pointer.h | 1358 +++++++++++ ext/librethinkdbxx/src/rapidjson/prettywriter.h | 249 ++ ext/librethinkdbxx/src/rapidjson/rapidjson.h | 615 +++++ ext/librethinkdbxx/src/rapidjson/reader.h | 1879 ++++++++++++++ ext/librethinkdbxx/src/rapidjson/schema.h | 2006 +++++++++++++++ ext/librethinkdbxx/src/rapidjson/stream.h | 179 ++ ext/librethinkdbxx/src/rapidjson/stringbuffer.h | 117 + ext/librethinkdbxx/src/rapidjson/writer.h | 609 +++++ ext/librethinkdbxx/src/term.cc | 285 +++ ext/librethinkdbxx/src/term.h | 592 +++++ ext/librethinkdbxx/src/types.cc | 47 + ext/librethinkdbxx/src/types.h | 53 + ext/librethinkdbxx/src/utils.cc | 153 ++ ext/librethinkdbxx/src/utils.h | 19 + ext/librethinkdbxx/test/bench.cc | 58 + ext/librethinkdbxx/test/gen_index_cxx.py | 11 + ext/librethinkdbxx/test/test.cc | 114 + ext/librethinkdbxx/test/testlib.cc | 356 +++ ext/librethinkdbxx/test/testlib.h | 231 ++ ext/librethinkdbxx/test/upstream/aggregation.yaml | 575 +++++ ext/librethinkdbxx/test/upstream/arity.yaml | 316 +++ .../test/upstream/changefeeds/edge.yaml | 142 ++ .../test/upstream/changefeeds/geo.rb.yaml | 27 + .../test/upstream/changefeeds/idxcopy.yaml | 38 + .../test/upstream/changefeeds/include_states.yaml | 58 + .../test/upstream/changefeeds/point.yaml | 147 ++ .../test/upstream/changefeeds/sindex.yaml | 50 + .../test/upstream/changefeeds/squash.yaml | 62 + .../test/upstream/changefeeds/table.yaml | 101 + ext/librethinkdbxx/test/upstream/control.yaml | 297 +++ ext/librethinkdbxx/test/upstream/datum/array.yaml | 133 + ext/librethinkdbxx/test/upstream/datum/binary.yaml | 363 +++ ext/librethinkdbxx/test/upstream/datum/bool.yaml | 47 + ext/librethinkdbxx/test/upstream/datum/null.yaml | 18 + ext/librethinkdbxx/test/upstream/datum/number.yaml | 125 + ext/librethinkdbxx/test/upstream/datum/object.yaml | 85 + ext/librethinkdbxx/test/upstream/datum/string.yaml | 329 +++ ext/librethinkdbxx/test/upstream/datum/typeof.yaml | 14 + ext/librethinkdbxx/test/upstream/datum/uuid.yaml | 20 + ext/librethinkdbxx/test/upstream/default.yaml | 270 ++ .../test/upstream/geo/constructors.yaml | 64 + ext/librethinkdbxx/test/upstream/geo/geojson.yaml | 31 + ext/librethinkdbxx/test/upstream/geo/indexing.yaml | 208 ++ .../test/upstream/geo/intersection_inclusion.yaml | 119 + .../test/upstream/geo/operations.yaml | 97 + .../test/upstream/geo/primitives.yaml | 50 + ext/librethinkdbxx/test/upstream/joins.yaml | 133 + ext/librethinkdbxx/test/upstream/json.yaml | 74 + ext/librethinkdbxx/test/upstream/limits.yaml | 128 + ext/librethinkdbxx/test/upstream/match.yaml | 38 + .../test/upstream/math_logic/add.yaml | 65 + .../test/upstream/math_logic/aliases.yaml | 46 + .../test/upstream/math_logic/comparison.yaml | 477 ++++ .../test/upstream/math_logic/div.yaml | 52 + .../test/upstream/math_logic/floor_ceil_round.yaml | 114 + .../test/upstream/math_logic/logic.yaml | 169 ++ .../test/upstream/math_logic/math.yaml | 11 + .../test/upstream/math_logic/mod.yaml | 34 + .../test/upstream/math_logic/mul.yaml | 60 + .../test/upstream/math_logic/sub.yaml | 37 + .../test/upstream/meta/composite.py.yaml | 14 + ext/librethinkdbxx/test/upstream/meta/dbs.yaml | 51 + ext/librethinkdbxx/test/upstream/meta/table.yaml | 365 +++ .../test/upstream/mutation/atomic_get_set.yaml | 93 + .../test/upstream/mutation/delete.yaml | 50 + .../test/upstream/mutation/insert.yaml | 239 ++ .../test/upstream/mutation/replace.yaml | 110 + .../test/upstream/mutation/sync.yaml | 52 + .../test/upstream/mutation/update.yaml | 170 ++ ext/librethinkdbxx/test/upstream/parsePolyglot.py | 164 ++ ext/librethinkdbxx/test/upstream/polymorphism.yaml | 33 + ext/librethinkdbxx/test/upstream/random.yaml | 180 ++ ext/librethinkdbxx/test/upstream/range.yaml | 53 + .../test/upstream/regression/1001.yaml | 20 + .../test/upstream/regression/1005.yaml | 19 + .../test/upstream/regression/1023.yaml | 65 + .../test/upstream/regression/1081.yaml | 39 + .../test/upstream/regression/1132.yaml | 4 + .../test/upstream/regression/1133.yaml | 19 + .../test/upstream/regression/1155.yaml | 5 + .../test/upstream/regression/1179.yaml | 26 + .../test/upstream/regression/1468.yaml | 7 + .../test/upstream/regression/1789.yaml | 22 + .../test/upstream/regression/2052.yaml | 10 + .../test/upstream/regression/2399.rb.yaml | 45 + .../test/upstream/regression/2639.rb.yaml | 8 + .../test/upstream/regression/2696.yaml | 6 + .../test/upstream/regression/2697.yaml | 31 + .../test/upstream/regression/2709.yaml | 21 + .../test/upstream/regression/2710.yaml | 6 + .../test/upstream/regression/2766.yaml | 25 + .../test/upstream/regression/2767.yaml | 20 + .../test/upstream/regression/2774.yaml | 99 + .../test/upstream/regression/2838.py.yaml | 16 + .../test/upstream/regression/2930.yaml | 17 + .../test/upstream/regression/3057.yaml | 10 + .../test/upstream/regression/3059.yaml | 7 + .../test/upstream/regression/309.yaml | 15 + .../test/upstream/regression/3444.yaml | 38 + .../test/upstream/regression/3449.js.yaml | 21 + .../test/upstream/regression/354.yaml | 20 + .../test/upstream/regression/3637.yaml | 51 + .../test/upstream/regression/370.yaml | 19 + .../test/upstream/regression/3745.yaml | 17 + .../test/upstream/regression/3759.yaml | 26 + .../test/upstream/regression/4030.yaml | 48 + .../test/upstream/regression/4063.js.yaml | 9 + .../test/upstream/regression/4132.yaml | 12 + .../test/upstream/regression/4146.yaml | 14 + .../test/upstream/regression/4431.yaml | 10 + .../test/upstream/regression/4462.yaml | 24 + .../test/upstream/regression/4465.py.yaml.disabled | 8 + .../test/upstream/regression/4501.yaml | 5 + .../test/upstream/regression/453.yaml | 16 + .../test/upstream/regression/4582.yaml | 9 + .../test/upstream/regression/4591.yaml | 4 + .../test/upstream/regression/46.yaml | 11 + .../test/upstream/regression/469.yaml | 139 ++ .../test/upstream/regression/4729.yaml | 13 + .../test/upstream/regression/5092.js.yaml | 14 + .../test/upstream/regression/5130.js.yaml | 5 + .../test/upstream/regression/522.py.yaml | 7 + .../test/upstream/regression/5222.py.yaml | 23 + .../test/upstream/regression/5241.yaml | 28 + .../test/upstream/regression/5383.rb.yaml | 20 + .../test/upstream/regression/5438.yaml | 16 + .../test/upstream/regression/545.yaml | 15 + .../test/upstream/regression/546.yaml | 27 + .../test/upstream/regression/5481.py.yaml | 5 + .../test/upstream/regression/5535.rb.yaml | 12 + .../test/upstream/regression/5542.py.yaml | 11 + .../test/upstream/regression/568.yaml | 10 + .../test/upstream/regression/578.yaml | 25 + .../test/upstream/regression/579.yaml | 15 + .../test/upstream/regression/619.yaml | 15 + .../test/upstream/regression/665.yaml | 19 + .../test/upstream/regression/678.yaml | 10 + .../test/upstream/regression/718.yaml | 7 + .../test/upstream/regression/730.yaml | 4 + .../test/upstream/regression/757.yaml | 9 + .../test/upstream/regression/763.js.yaml | 33 + .../test/upstream/regression/767.yaml | 10 + .../test/upstream/regression/831.yaml | 5 + ext/librethinkdbxx/test/upstream/selection.yaml | 355 +++ ext/librethinkdbxx/test/upstream/sindex/api.yaml | 1348 ++++++++++ .../test/upstream/sindex/nullsinstrings.yaml | 21 + .../test/upstream/sindex/status.yaml | 76 + .../test/upstream/sindex/trunc_array.rb.yaml | 70 + .../test/upstream/sindex/truncation.rb.yaml | 57 + ext/librethinkdbxx/test/upstream/timeout.yaml | 39 + ext/librethinkdbxx/test/upstream/times/api.yaml | 144 ++ .../test/upstream/times/constructors.yaml | 96 + ext/librethinkdbxx/test/upstream/times/index.yaml | 151 ++ .../test/upstream/times/portions.yaml | 45 + ext/librethinkdbxx/test/upstream/times/shim.yaml | 22 + .../test/upstream/times/time_arith.yaml | 197 ++ .../test/upstream/times/timezones.yaml | 87 + .../test/upstream/transform/array.yaml | 303 +++ .../test/upstream/transform/fold.yaml | 49 + .../test/upstream/transform/map.yaml | 106 + .../test/upstream/transform/object.yaml | 147 ++ .../test/upstream/transform/table.yaml | 19 + .../test/upstream/transform/unordered_map.yaml | 63 + .../test/upstream/transformation.yaml | 543 +++++ ext/librethinkdbxx/test/yaml_to_cxx.py | 467 ++++ node/InetAddress.hpp | 7 + osdep/OSUtils.cpp | 15 + osdep/OSUtils.hpp | 1 + 219 files changed, 33295 insertions(+) create mode 100644 controller/RethinkDB.cpp create mode 100644 controller/RethinkDB.hpp create mode 100644 ext/librethinkdbxx/.travis.yml create mode 100644 ext/librethinkdbxx/COPYRIGHT create mode 100644 ext/librethinkdbxx/Makefile create mode 100644 ext/librethinkdbxx/README.md create mode 100644 ext/librethinkdbxx/reql/add_docs.py create mode 100644 ext/librethinkdbxx/reql/gen.py create mode 100644 ext/librethinkdbxx/reql/python_docs.txt create mode 100644 ext/librethinkdbxx/reql/ql2.proto create mode 100644 ext/librethinkdbxx/src/connection.cc create mode 100644 ext/librethinkdbxx/src/connection.h create mode 100644 ext/librethinkdbxx/src/connection_p.h create mode 100644 ext/librethinkdbxx/src/cursor.cc create mode 100644 ext/librethinkdbxx/src/cursor.h create mode 100644 ext/librethinkdbxx/src/cursor_p.h create mode 100644 ext/librethinkdbxx/src/datum.cc create mode 100644 ext/librethinkdbxx/src/datum.h create mode 100644 ext/librethinkdbxx/src/error.h create mode 100644 ext/librethinkdbxx/src/exceptions.h create mode 100644 ext/librethinkdbxx/src/json.cc create mode 100644 ext/librethinkdbxx/src/json_p.h create mode 100644 ext/librethinkdbxx/src/rapidjson-config.h create mode 100644 ext/librethinkdbxx/src/rapidjson/allocators.h create mode 100644 ext/librethinkdbxx/src/rapidjson/document.h create mode 100644 ext/librethinkdbxx/src/rapidjson/encodedstream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/encodings.h create mode 100644 ext/librethinkdbxx/src/rapidjson/error/en.h create mode 100644 ext/librethinkdbxx/src/rapidjson/error/error.h create mode 100644 ext/librethinkdbxx/src/rapidjson/filereadstream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/filewritestream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/fwd.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/biginteger.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/diyfp.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/dtoa.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/ieee754.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/itoa.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/meta.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/pow10.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/regex.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/stack.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/strfunc.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/strtod.h create mode 100644 ext/librethinkdbxx/src/rapidjson/internal/swap.h create mode 100644 ext/librethinkdbxx/src/rapidjson/istreamwrapper.h create mode 100644 ext/librethinkdbxx/src/rapidjson/memorybuffer.h create mode 100644 ext/librethinkdbxx/src/rapidjson/memorystream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h create mode 100644 ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h create mode 100644 ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h create mode 100644 ext/librethinkdbxx/src/rapidjson/pointer.h create mode 100644 ext/librethinkdbxx/src/rapidjson/prettywriter.h create mode 100644 ext/librethinkdbxx/src/rapidjson/rapidjson.h create mode 100644 ext/librethinkdbxx/src/rapidjson/reader.h create mode 100644 ext/librethinkdbxx/src/rapidjson/schema.h create mode 100644 ext/librethinkdbxx/src/rapidjson/stream.h create mode 100644 ext/librethinkdbxx/src/rapidjson/stringbuffer.h create mode 100644 ext/librethinkdbxx/src/rapidjson/writer.h create mode 100644 ext/librethinkdbxx/src/term.cc create mode 100644 ext/librethinkdbxx/src/term.h create mode 100644 ext/librethinkdbxx/src/types.cc create mode 100644 ext/librethinkdbxx/src/types.h create mode 100644 ext/librethinkdbxx/src/utils.cc create mode 100644 ext/librethinkdbxx/src/utils.h create mode 100644 ext/librethinkdbxx/test/bench.cc create mode 100644 ext/librethinkdbxx/test/gen_index_cxx.py create mode 100644 ext/librethinkdbxx/test/test.cc create mode 100644 ext/librethinkdbxx/test/testlib.cc create mode 100644 ext/librethinkdbxx/test/testlib.h create mode 100644 ext/librethinkdbxx/test/upstream/aggregation.yaml create mode 100644 ext/librethinkdbxx/test/upstream/arity.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/point.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml create mode 100644 ext/librethinkdbxx/test/upstream/changefeeds/table.yaml create mode 100644 ext/librethinkdbxx/test/upstream/control.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/array.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/binary.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/bool.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/null.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/number.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/object.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/string.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/typeof.yaml create mode 100644 ext/librethinkdbxx/test/upstream/datum/uuid.yaml create mode 100644 ext/librethinkdbxx/test/upstream/default.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/constructors.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/geojson.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/indexing.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/operations.yaml create mode 100644 ext/librethinkdbxx/test/upstream/geo/primitives.yaml create mode 100644 ext/librethinkdbxx/test/upstream/joins.yaml create mode 100644 ext/librethinkdbxx/test/upstream/json.yaml create mode 100644 ext/librethinkdbxx/test/upstream/limits.yaml create mode 100644 ext/librethinkdbxx/test/upstream/match.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/add.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/div.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/logic.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/math.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/mod.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/mul.yaml create mode 100644 ext/librethinkdbxx/test/upstream/math_logic/sub.yaml create mode 100644 ext/librethinkdbxx/test/upstream/meta/composite.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/meta/dbs.yaml create mode 100644 ext/librethinkdbxx/test/upstream/meta/table.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/delete.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/insert.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/replace.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/sync.yaml create mode 100644 ext/librethinkdbxx/test/upstream/mutation/update.yaml create mode 100755 ext/librethinkdbxx/test/upstream/parsePolyglot.py create mode 100644 ext/librethinkdbxx/test/upstream/polymorphism.yaml create mode 100644 ext/librethinkdbxx/test/upstream/random.yaml create mode 100644 ext/librethinkdbxx/test/upstream/range.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1001.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1005.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1023.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1081.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1132.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1133.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1155.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1179.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1468.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/1789.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2052.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2696.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2697.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2709.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2710.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2766.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2767.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2774.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2838.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/2930.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3057.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3059.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/309.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3444.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3449.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/354.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3637.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/370.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3745.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/3759.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4030.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4063.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4132.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4146.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4431.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4462.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4465.py.yaml.disabled create mode 100644 ext/librethinkdbxx/test/upstream/regression/4501.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/453.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4582.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4591.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/46.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/469.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/4729.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5092.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5130.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/522.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5222.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5241.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5383.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5438.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/545.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/546.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5481.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5535.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/5542.py.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/568.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/578.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/579.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/619.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/665.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/678.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/718.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/730.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/757.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/763.js.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/767.yaml create mode 100644 ext/librethinkdbxx/test/upstream/regression/831.yaml create mode 100644 ext/librethinkdbxx/test/upstream/selection.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/api.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/nullsinstrings.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/status.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/trunc_array.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/sindex/truncation.rb.yaml create mode 100644 ext/librethinkdbxx/test/upstream/timeout.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/api.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/constructors.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/index.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/portions.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/shim.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/time_arith.yaml create mode 100644 ext/librethinkdbxx/test/upstream/times/timezones.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/array.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/fold.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/map.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/object.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/table.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transform/unordered_map.yaml create mode 100644 ext/librethinkdbxx/test/upstream/transformation.yaml create mode 100644 ext/librethinkdbxx/test/yaml_to_cxx.py (limited to 'node') diff --git a/.gitignore b/.gitignore index 61dd1c85..936de092 100755 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,7 @@ build/ !default.perspectivev3 *.xccheckout xcuserdata/ +ext/librethinkdbxx/build +.vscode +__pycache__ +*~ diff --git a/controller/RethinkDB.cpp b/controller/RethinkDB.cpp new file mode 100644 index 00000000..5d93bf72 --- /dev/null +++ b/controller/RethinkDB.cpp @@ -0,0 +1,308 @@ +#include "RethinkDB.hpp" + +#include +#include +#include + +#include "../ext/librethinkdbxx/build/include/rethinkdb.h" + +namespace R = RethinkDB; +using nlohmann::json; + +namespace ZeroTier { + +RethinkDB::RethinkDB(const Address &myAddress,const char *host,const int port,const char *db,const char *auth) : + _myAddress(myAddress), + _host(host ? host : "127.0.0.1"), + _db(db), + _auth(auth ? auth : ""), + _port((port > 0) ? port : 28015), + _ready(2), // two tables need to be synchronized before we're ready + _run(1) +{ + _readyLock.lock(); + + { + char tmp[32]; + _myAddress.toString(tmp); + _myAddressStr = tmp; + } + + _membersDbWatcher = std::thread([this]() { + while (_run == 1) { + try { + auto rdb = R::connect(this->_host,this->_port,this->_auth); + if (rdb) { + _membersDbWatcherConnection = (void *)rdb.get(); + auto cur = R::db(this->_db).table("Member").get_all(this->_myAddressStr,R::optargs("index","controllerId")).changes(R::optargs("squash",0.1,"include_initial",true,"include_types",true,"include_states",true)).run(*rdb); + while (cur.has_next()) { + if (_run != 1) break; + json tmp(json::parse(cur.next().as_json())); + if ((tmp["type"] == "state")&&(tmp["state"] == "ready")) { + if (--this->_ready == 0) + this->_readyLock.unlock(); + } else { + try { + this->_memberChanged(tmp["old_val"],tmp["new_val"]); + } catch ( ... ) {} // ignore bad records + } + } + } + } catch (std::exception &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.what()); + } catch (R::Error &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.message.c_str()); + } catch ( ... ) { + fprintf(stderr,"ERROR: controller RethinkDB: unknown exception" ZT_EOL_S); + } + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + }); + + _networksDbWatcher = std::thread([this]() { + while (_run == 1) { + try { + auto rdb = R::connect(this->_host,this->_port,this->_auth); + if (rdb) { + _membersDbWatcherConnection = (void *)rdb.get(); + auto cur = R::db(this->_db).table("Network").get_all(this->_myAddressStr,R::optargs("index","controllerId")).changes(R::optargs("squash",0.1,"include_initial",true,"include_types",true,"include_states",true)).run(*rdb); + while (cur.has_next()) { + if (_run != 1) break; + json tmp(json::parse(cur.next().as_json())); + if ((tmp["type"] == "state")&&(tmp["state"] == "ready")) { + if (--this->_ready == 0) + this->_readyLock.unlock(); + } else { + try { + this->_networkChanged(tmp["old_val"],tmp["new_val"]); + } catch ( ... ) {} // ignore bad records + } + } + } + } catch (std::exception &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.what()); + } catch (R::Error &e) { + fprintf(stderr,"ERROR: controller RethinkDB: %s" ZT_EOL_S,e.message.c_str()); + } catch ( ... ) { + fprintf(stderr,"ERROR: controller RethinkDB: unknown exception" ZT_EOL_S); + } + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + }); +} + +RethinkDB::~RethinkDB() +{ + // FIXME: not totally safe but will generally work, and only happens on shutdown anyway + _run = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (_membersDbWatcherConnection) + ((R::Connection *)_membersDbWatcherConnection)->close(); + if (_networksDbWatcherConnection) + ((R::Connection *)_networksDbWatcherConnection)->close(); + _membersDbWatcher.join(); + _networksDbWatcher.join(); +} + +inline bool RethinkDB::get(const uint64_t networkId,nlohmann::json &network) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + network = nw->config; + + return true; +} + +inline bool RethinkDB::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + auto m = nw->members.find(memberId); + if (m == nw->members.end()) + return false; + network = nw->config; + member = m->second; + _fillSummaryInfo(nw,info); + + return true; +} + +inline bool RethinkDB::get(const uint64_t networkId,nlohmann::json &network,std::vector &members) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + network = nw->config; + for(auto m=nw->members.begin();m!=nw->members.end();++m) + members.push_back(m->second); + + return true; +} + +inline bool RethinkDB::summary(const uint64_t networkId,NetworkSummaryInfo &info) +{ + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + auto nwi = _networks.find(networkId); + if (nwi == _networks.end()) + return false; + nw = nwi->second; + } + + std::lock_guard l2(nw->lock); + _fillSummaryInfo(nw,info); + + return true; +} + +void RethinkDB::_memberChanged(nlohmann::json &old,nlohmann::json &member) +{ + uint64_t memberId = 0; + uint64_t networkId = 0; + std::shared_ptr<_Network> nw; + + if (old.is_object()) { + json &config = old["config"]; + if (config.is_object()) { + memberId = OSUtils::jsonIntHex(config["id"],0ULL); + networkId = OSUtils::jsonIntHex(config["nwid"],0ULL); + if ((memberId)&&(networkId)) { + { + std::lock_guard l(_networks_l); + auto nw2 = _networks.find(networkId); + if (nw2 != _networks.end()) + nw = nw2->second; + } + if (nw) { + std::lock_guard l(nw->lock); + if (OSUtils::jsonBool(config["activeBridge"],false)) + nw->activeBridgeMembers.erase(memberId); + if (OSUtils::jsonBool(config["authorized"],false)) + nw->authorizedMembers.erase(memberId); + json &ips = config["ipAssignments"]; + if (ips.is_array()) { + for(unsigned long i=0;iallocatedIps.erase(ipa); + } + } + } + } + } + } + } + + if (member.is_object()) { + json &config = member["config"]; + if (config.is_object()) { + if (!nw) { + memberId = OSUtils::jsonIntHex(config["id"],0ULL); + networkId = OSUtils::jsonIntHex(config["nwid"],0ULL); + if ((!memberId)||(!networkId)) + return; + std::lock_guard l(_networks_l); + std::shared_ptr<_Network> &nw2 = _networks[networkId]; + if (!nw2) + nw2.reset(new _Network); + nw = nw2; + } + std::lock_guard l(nw->lock); + + nw->members[memberId] = config; + + if (OSUtils::jsonBool(config["activeBridge"],false)) + nw->activeBridgeMembers.insert(memberId); + const bool isAuth = OSUtils::jsonBool(config["authorized"],false); + if (isAuth) + nw->authorizedMembers.insert(memberId); + json &ips = config["ipAssignments"]; + if (ips.is_array()) { + for(unsigned long i=0;iallocatedIps.insert(ipa); + } + } + } + + if (!isAuth) { + const int64_t ldt = (int64_t)OSUtils::jsonInt(config["lastDeauthorizedTime"],0ULL); + if (ldt > nw->mostRecentDeauthTime) + nw->mostRecentDeauthTime = ldt; + } + } + } +} + +void RethinkDB::_networkChanged(nlohmann::json &old,nlohmann::json &network) +{ + if (network.is_object()) { + json &config = network["config"]; + if (config.is_object()) { + const std::string ids = config["id"]; + const uint64_t id = Utils::hexStrToU64(ids.c_str()); + if (id) { + std::shared_ptr<_Network> nw; + { + std::lock_guard l(_networks_l); + std::shared_ptr<_Network> &nw2 = _networks[id]; + if (!nw2) + nw2.reset(new _Network); + nw = nw2; + } + std::lock_guard l2(nw->lock); + nw->config = config; + } + } + } else if (old.is_object()) { + const std::string ids = old["id"]; + const uint64_t id = Utils::hexStrToU64(ids.c_str()); + if (id) { + std::lock_guard l(_networks_l); + _networks.erase(id); + } + } +} + +} // namespace ZeroTier + +/* +int main(int argc,char **argv) +{ + ZeroTier::RethinkDB db(ZeroTier::Address(0x8056c2e21cULL),"10.6.6.188",28015,"ztc",""); + db.waitForReady(); + printf("ready.\n"); + pause(); +} +*/ diff --git a/controller/RethinkDB.hpp b/controller/RethinkDB.hpp new file mode 100644 index 00000000..7ed0e0a8 --- /dev/null +++ b/controller/RethinkDB.hpp @@ -0,0 +1,101 @@ +#ifndef ZT_CONTROLLER_RETHINKDB_HPP +#define ZT_CONTROLLER_RETHINKDB_HPP + +#include "../node/Constants.hpp" +#include "../node/Address.hpp" +#include "../node/InetAddress.hpp" +#include "../osdep/OSUtils.hpp" + +#include +#include +#include +#include +#include +#include + +#include "../ext/json/json.hpp" + +namespace ZeroTier +{ + +class RethinkDB +{ +public: + struct NetworkSummaryInfo + { + NetworkSummaryInfo() : authorizedMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} + std::vector
activeBridges; + std::vector allocatedIps; + unsigned long authorizedMemberCount; + unsigned long totalMemberCount; + int64_t mostRecentDeauthTime; + }; + + RethinkDB(const Address &myAddress,const char *host,const int port,const char *db,const char *auth); + ~RethinkDB(); + + inline bool ready() const { return (_ready <= 0); } + + inline void waitForReady() const + { + while (_ready > 0) { + _readyLock.lock(); + _readyLock.unlock(); + } + } + + bool get(const uint64_t networkId,nlohmann::json &network); + bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info); + bool get(const uint64_t networkId,nlohmann::json &network,std::vector &members); + bool summary(const uint64_t networkId,NetworkSummaryInfo &info); + +private: + struct _Network + { + _Network() : mostRecentDeauthTime(0) {} + nlohmann::json config; + std::unordered_map members; + std::unordered_set activeBridgeMembers; + std::unordered_set authorizedMembers; + std::unordered_set allocatedIps; + int64_t mostRecentDeauthTime; + std::mutex lock; + }; + + void _memberChanged(nlohmann::json &old,nlohmann::json &member); + void _networkChanged(nlohmann::json &old,nlohmann::json &network); + + inline void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info) + { + for(auto ab=nw->activeBridgeMembers.begin();ab!=nw->activeBridgeMembers.end();++ab) + info.activeBridges.push_back(Address(*ab)); + for(auto ip=nw->allocatedIps.begin();ip!=nw->allocatedIps.end();++ip) + info.allocatedIps.push_back(*ip); + info.authorizedMemberCount = (unsigned long)nw->authorizedMembers.size(); + info.totalMemberCount = (unsigned long)nw->members.size(); + info.mostRecentDeauthTime = nw->mostRecentDeauthTime; + } + + const Address _myAddress; + std::string _myAddressStr; + std::string _host; + std::string _db; + std::string _auth; + const int _port; + + void *_networksDbWatcherConnection; + void *_membersDbWatcherConnection; + std::thread _networksDbWatcher; + std::thread _membersDbWatcher; + + std::unordered_map< uint64_t,std::shared_ptr<_Network> > _networks; + std::mutex _networks_l; + + mutable std::mutex _readyLock; // locked until ready + std::atomic _ready; + std::atomic _run; +}; + +} // namespace ZeroTier + +#endif diff --git a/ext/librethinkdbxx/.travis.yml b/ext/librethinkdbxx/.travis.yml new file mode 100644 index 00000000..b306a410 --- /dev/null +++ b/ext/librethinkdbxx/.travis.yml @@ -0,0 +1,11 @@ +sudo: required +dist: trusty + +python: + - "3.4.3" + +addons: + rethinkdb: "2.3" + +script: + - make test diff --git a/ext/librethinkdbxx/COPYRIGHT b/ext/librethinkdbxx/COPYRIGHT new file mode 100644 index 00000000..c25145d5 --- /dev/null +++ b/ext/librethinkdbxx/COPYRIGHT @@ -0,0 +1,16 @@ +RethinkDB Language Drivers + +Copyright 2010-2012 RethinkDB + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this product except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/ext/librethinkdbxx/Makefile b/ext/librethinkdbxx/Makefile new file mode 100644 index 00000000..ba319c16 --- /dev/null +++ b/ext/librethinkdbxx/Makefile @@ -0,0 +1,126 @@ +# Customisable build settings + +CXX ?= clang++ +CXXFLAGS ?= +INCLUDE_PYTHON_DOCS ?= no +DEBUG ?= no +PYTHON ?= python3 + +# Required build settings + +ifneq (no,$(DEBUG)) + CXXFLAGS += -ggdb +else + CXXFLAGS += -O3 # -flto +endif + +CXXFLAGS += -std=c++11 -I'build/gen' -Wall -pthread -fPIC + +prefix ?= /usr +DESTDIR ?= + +.DELETE_ON_ERROR: +SHELL := /bin/bash + +modules := connection datum json term cursor types utils +headers := utils error exceptions types datum connection cursor term + +o_files := $(patsubst %, build/obj/%.o, $(modules)) +d_files := $(patsubst %, build/dep/%.d, $(modules)) + +skip_tests := regression/1133 regression/767 regression/1005 # python-only +skip_tests += arity # arity errors are compile-time +skip_tests += geo # geo types not implemented yet +skip_tests += limits # possibly broken tests: https://github.com/rethinkdb/rethinkdb/issues/5940 + +upstream_tests := \ + $(filter-out %.rb.%, \ + $(filter-out $(patsubst %,test/upstream/%%, $(skip_tests)), \ + $(filter test/upstream/$(test_filter)%, \ + $(shell find test/upstream -name '*.yaml' | egrep -v '.(rb|js).yaml$$')))) +upstream_tests_cc := $(patsubst %.yaml, build/tests/%.cc, $(upstream_tests)) +upstream_tests_o := $(patsubst %.cc, %.o, $(upstream_tests_cc)) + +.PRECIOUS: $(upstream_tests_cc) $(upstream_tests_o) + +default: build/librethinkdb++.a build/include/rethinkdb.h build/librethinkdb++.so + +all: default build/test + +build/librethinkdb++.a: $(o_files) + ar rcs $@ $^ + +build/librethinkdb++.so: $(o_files) + $(CXX) -o $@ $(CXXFLAGS) -shared $^ + +build/obj/%.o: src/%.cc build/gen/protocol_defs.h + @mkdir -p $(dir $@) + @mkdir -p $(dir build/dep/$*.d) + $(CXX) -o $@ $(CXXFLAGS) -c $< -MP -MQ $@ -MD -MF build/dep/$*.d + +build/gen/protocol_defs.h: reql/ql2.proto reql/gen.py | build/gen/. + $(PYTHON) reql/gen.py $< > $@ + +clean: + rm -rf build + +ifneq (no,$(INCLUDE_PYTHON_DOCS)) +build/include/rethinkdb.h: build/rethinkdb.nodocs.h reql/add_docs.py reql/python_docs.txt | build/include/. + $(PYTHON) reql/add_docs.py reql/python_docs.txt < $< > $@ +else +build/include/rethinkdb.h: build/rethinkdb.nodocs.h | build/include/. + cp $< $@ +endif + +build/rethinkdb.nodocs.h: build/gen/protocol_defs.h $(patsubst %, src/%.h, $(headers)) + ( echo "// Auto-generated file, built from $^"; \ + echo '#pragma once'; \ + cat $^ | \ + grep -v '^#pragma once' | \ + grep -v '^#include "'; \ + ) > $@ + +build/tests/%.cc: %.yaml test/yaml_to_cxx.py + @mkdir -p $(dir $@) + $(PYTHON) test/yaml_to_cxx.py $< > $@ + +build/tests/upstream_tests.cc: $(upstream_tests) test/gen_index_cxx.py FORCE | build/tests/. + @echo '$(PYTHON) test/gen_index_cxx.py $(wordlist 1,5,$(upstream_tests)) ... > $@' + @$(PYTHON) test/gen_index_cxx.py $(upstream_tests) > $@ + +build/tests/%.o: build/tests/%.cc build/include/rethinkdb.h test/testlib.h | build/tests/. + $(CXX) -o $@ $(CXXFLAGS) -isystem build/include -I test -c $< -Wno-unused-variable + +build/tests/%.o: test/%.cc test/testlib.h build/include/rethinkdb.h | build/tests/. + $(CXX) -o $@ $(CXXFLAGS) -isystem build/include -I test -c $< + +build/test: build/tests/testlib.o build/tests/test.o build/tests/upstream_tests.o $(upstream_tests_o) build/librethinkdb++.a + @echo $(CXX) -o $@ $(CXXFLAGS) $(wordlist 1,5,$^) ... + @$(CXX) -o $@ $(CXXFLAGS) build/librethinkdb++.a $^ + +.PHONY: test +test: build/test + build/test + +build/bench: build/tests/bench.o build/librethinkdb++.a + @$(CXX) -o $@ $(CXXFLAGS) -isystem build/include build/librethinkdb++.a $^ + +.PHONY: bench +bench: build/bench + build/bench + +.PHONY: install +install: build/librethinkdb++.a build/include/rethinkdb.h build/librethinkdb++.so + install -m755 -d $(DESTDIR)$(prefix)/lib + install -m755 -d $(DESTDIR)$(prefix)/include + install -m644 build/librethinkdb++.a $(DESTDIR)$(prefix)/lib/librethinkdb++.a + install -m644 build/librethinkdb++.so $(DESTDIR)$(prefix)/lib/librethinkdb++.so + install -m644 build/include/rethinkdb.h $(DESTDIR)$(prefix)/include/rethinkdb.h + +%/.: + mkdir -p $* + +.PHONY: FORCE +FORCE: + +-include $(d_files) diff --git a/ext/librethinkdbxx/README.md b/ext/librethinkdbxx/README.md new file mode 100644 index 00000000..92fa9136 --- /dev/null +++ b/ext/librethinkdbxx/README.md @@ -0,0 +1,72 @@ +# RethinkDB driver for C++ + +This driver is compatible with RethinkDB 2.0. It is based on the +official RethinkDB Python driver. + +* [RethinkDB server](http://rethinkdb.com/) +* [RethinkDB API docs](http://rethinkdb.com/api/python/) + +## Example + +``` +#include +#include +#include + +namespace R = RethinkDB; + +int main() { + std::unique_ptr conn = R::connect("localhost", 28015); + R::Cursor cursor = R::table("users").filter(R::row["age"] > 14).run(*conn); + for (R::Datum& user : cursor) { + printf("%s\n", user.as_json().c_str()); + } +} +``` + +## Build + +Requires a modern C++ compiler. to build and install, run: + +``` +make +make install +``` + +Will build `include/rethinkdb.h`, `librethinkdb++.a` and +`librethinkdb++.so` into the `build/` directory. + +To include documentation from the Python driver in the header file, +pass the following argument to make. + +``` +make INCLUDE_PYTHON_DOCS=yes +``` + +To build in debug mode: + +``` +make DEBUG=yes +``` + +To install to a specific location: + +``` +make install prefix=/usr/local DESTDIR= +``` + +## Status + +Still in early stages of development. + +## Tests + +This driver is tested against the upstream ReQL tests from the +RethinkDB repo, which are programmatically translated from Python to +C++. As of 34dc13c, all tests pass: + +``` +$ make test +... +SUCCESS: 2053 tests passed +``` diff --git a/ext/librethinkdbxx/reql/add_docs.py b/ext/librethinkdbxx/reql/add_docs.py new file mode 100644 index 00000000..67f08df8 --- /dev/null +++ b/ext/librethinkdbxx/reql/add_docs.py @@ -0,0 +1,80 @@ +from sys import stdin, stderr, stdout, argv +from re import match, sub + +docs = {} + +for line in open(argv[1]): + res = match('^\t\(([^,]*), (.*)\),$', line) + if res: + fullname = res.group(1) + docs[fullname.split('.')[-1]] = eval(res.group(2)).decode('utf-8') + +translate_name = { + 'name': None, + 'delete_': 'delete', + 'union_': 'union', + 'operator[]': '__getitem__', + 'operator+': '__add__', + 'operator-': '__sub__', + 'operator*': '__mul__', + 'operator/': '__div__', + 'operator%': '__mod__', + 'operator&&': 'and_', + 'operator||': 'or_', + 'operator==': '__eq__', + 'operator!=': '__ne__', + 'operator>': '__gt__', + 'operator>=': '__ge__', + 'operator<': '__lt__', + 'operator<=': '__le__', + 'operator!': 'not_', + 'default_': 'default', + 'array': None, + 'desc': None, + 'asc': None, + 'maxval': None, + 'minval': None, + 'january': None, + 'february': None, + 'march': None, + 'april': None, + 'may': None, + 'june': None, + 'july': None, + 'august': None, + 'september': None, + 'october': None, + 'november': None, + 'december': None, + 'monday': None, + 'tuesday': None, + 'wednesday': None, + 'thursday': None, + 'friday': None, + 'saturday': None, + 'sunday': None, +} + +def print_docs(name, line): + py_name = translate_name.get(name, name) + if py_name in docs: + indent = match("^( *)", line).group(1) + stdout.write('\n') + # TODO: convert the examples to C++ + for line in docs[py_name].split('\n'): + stdout.write(indent + "// " + line + '\n') + elif py_name: + stderr.write('Warning: no docs for ' + py_name + ': ' + line) + +stdout.write('// Contains documentation copied as-is from the Python driver') + +for line in stdin: + res = match("^ *CO?[0-9_]+\(([^,)]+)|extern Query (\w+)|^ *// *(\$)doc\((\w+)\) *$", line) + if res: + name = res.group(1) or res.group(2) or res.group(4) + print_docs(name, line) + if not res.group(3): + stdout.write(line) + else: + stdout.write(line) + diff --git a/ext/librethinkdbxx/reql/gen.py b/ext/librethinkdbxx/reql/gen.py new file mode 100644 index 00000000..2b1fe9fc --- /dev/null +++ b/ext/librethinkdbxx/reql/gen.py @@ -0,0 +1,33 @@ +from sys import argv +from re import sub, finditer, VERBOSE + +def gen(defs): + indent = 0 + enum = False + def p(s): print(" " * (indent * 4) + s) + for item in finditer(""" + (?P message|enum) \\s+ (?P \\w+) \\s* \\{ | + (?P \\w+) \\s* = \\s* (?P \\w+) \\s* ; | + \\} + """, defs, flags=VERBOSE): + if item.group(0) == "}": + indent = indent - 1 + p("};" if enum else "}") + enum = False; + elif item.group('type') == 'enum': + p("enum class %s {" % item.group('name')) + indent = indent + 1 + enum = True + elif item.group('type') == 'message': + p("namespace %s {" % item.group('name')) + indent = indent + 1 + enum = False + else: + if enum: + p("%s = %s," % (item.group('var'), item.group('val'))) + +print("// Auto-generated by reql/gen.py") +print("#pragma once") +print("namespace RethinkDB { namespace Protocol {") +gen(sub("//.*", "", open(argv[1]).read())) +print("} }") diff --git a/ext/librethinkdbxx/reql/python_docs.txt b/ext/librethinkdbxx/reql/python_docs.txt new file mode 100644 index 00000000..41a5c84e --- /dev/null +++ b/ext/librethinkdbxx/reql/python_docs.txt @@ -0,0 +1,189 @@ +# This file was generated by _scripts/gen_python.py from the rethinkdb documentation in http://github.com/rethinkdb/docs +# hash: "3d13a937cfdacb7ffa3dab2ce3ebdf25bd3c192e" + +import rethinkdb + +docsSource = [ + + (rethinkdb.net.Connection.close, b'conn.close(noreply_wait=True)\n\nClose an open connection.\n\nClosing a connection normally waits until all outstanding requests have finished and then frees any open resources associated with the connection. By passing `False` to the `noreply_wait` optional argument, the connection will be closed immediately, possibly aborting any outstanding noreply writes.\n\nA noreply query is executed by passing the `noreply` option to the [run](http://rethinkdb.com/api/python/run/) command, indicating that `run()` should not wait for the query to complete before returning. You may also explicitly wait for a noreply query to complete by using the [noreply_wait](http://rethinkdb.com/api/python/noreply_wait) command.\n\n*Example* Close an open connection, waiting for noreply writes to finish.\n\n conn.close()\n\n*Example* Close an open connection immediately.\n\n conn.close(noreply_wait=False)\n'), + (rethinkdb.connect, b'r.connect(host="localhost", port=28015, db="test", auth_key="", timeout=20) -> connection\nr.connect(host) -> connection\n\nCreate a new connection to the database server. The keyword arguments are:\n\n- `host`: host of the RethinkDB instance. The default value is `localhost`.\n- `port`: the driver port, by default `28015`.\n- `db`: the database used if not explicitly specified in a query, by default `test`.\n- `auth_key`: the authentication key, by default the empty string.\n- `timeout`: timeout period in seconds for the connection to be opened (default `20`).\n\nIf the connection cannot be established, a `RqlDriverError` exception will be thrown.\n\nThe authentication key can be set from the RethinkDB command line tool. Once set, client connections must provide the key as an option to `run` in order to make the connection. For more information, read "Using the RethinkDB authentication system" in the documentation on [securing your cluster](http://rethinkdb.com/docs/security/).\n\n__Note:__ Currently, the Python driver is not thread-safe. Each thread or multiprocessing PID should be given its own connection object. (This is likely to change in a future release of RethinkDB; you can track issue [#2427](https://github.com/rethinkdb/rethinkdb/issues/2427) for progress.)\n\n*Example* Opens a connection using the default host and port but specifying the default database.\n\n conn = r.connect(db=\'marvel\')\n\n*Example* Opens a new connection to the database.\n\n conn = r.connect(host = \'localhost\',\n port = 28015,\n db = \'heroes\',\n auth_key = \'hunter2\')\n\n'), + (rethinkdb.net.Connection.noreply_wait, b'conn.noreply_wait()\n\n`noreply_wait` ensures that previous queries with the `noreply` flag have been processed\nby the server. Note that this guarantee only applies to queries run on the given connection.\n\n*Example* We have previously run queries with the `noreply` argument set to `True`. Now\nwait until the server has processed them.\n\n conn.noreply_wait()\n\n'), + (rethinkdb, b'r -> r\n\nThe top-level ReQL namespace.\n\n*Example* Setup your top-level namespace.\n\n import rethinkdb as r\n\n'), + (rethinkdb.net.Connection.reconnect, b'conn.reconnect(noreply_wait=True)\n\nClose and reopen a connection.\n\nClosing a connection normally waits until all outstanding requests have finished and then frees any open resources associated with the connection. By passing `False` to the `noreply_wait` optional argument, the connection will be closed immediately, possibly aborting any outstanding noreply writes.\n\nA noreply query is executed by passing the `noreply` option to the [run](http://rethinkdb.com/api/python/run/) command, indicating that `run()` should not wait for the query to complete before returning. You may also explicitly wait for a noreply query to complete by using the [noreply_wait](http://rethinkdb.com/api/python/noreply_wait) command.\n\n*Example* Cancel outstanding requests/queries that are no longer needed.\n\n conn.reconnect(noreply_wait=False)\n'), + (rethinkdb.net.Connection.repl, b"conn.repl()\n\nSet the default connection to make REPL use easier. Allows calling\n`.run()` on queries without specifying a connection.\n\n__Note:__ Avoid using `repl` in application code. RethinkDB connection objects are not thread-safe, and calls to `connect` from multiple threads may change the global connection object used by `repl`. Applications should specify connections explicitly.\n\n*Example* Set the default connection for the REPL, then call\n`run()` without specifying the connection.\n\n r.connect(db='marvel').repl()\n r.table('heroes').run()\n"), + (rethinkdb.ast.RqlQuery.run, b'query.run(conn, use_outdated=False, time_format=\'native\', profile=False, durability="hard") -> cursor\nquery.run(conn, use_outdated=False, time_format=\'native\', profile=False, durability="hard") -> object\n\nRun a query on a connection, returning either a single JSON result or\na cursor, depending on the query.\n\nThe optional arguments are:\n\n- `use_outdated`: whether or not outdated reads are OK (default: `False`).\n- `time_format`: what format to return times in (default: `\'native\'`).\n Set this to `\'raw\'` if you want times returned as JSON objects for exporting.\n- `profile`: whether or not to return a profile of the query\'s\n execution (default: `False`).\n- `durability`: possible values are `\'hard\'` and `\'soft\'`. In soft durability mode RethinkDB\nwill acknowledge the write immediately after receiving it, but before the write has\nbeen committed to disk.\n- `group_format`: what format to return `grouped_data` and `grouped_streams` in (default: `\'native\'`).\n Set this to `\'raw\'` if you want the raw pseudotype.\n- `noreply`: set to `True` to not receive the result object or cursor and return immediately.\n- `db`: the database to run this query against as a string. The default is the database specified in the `db` parameter to [connect](http://rethinkdb.com/api/python/connect/) (which defaults to `test`). The database may also be specified with the [db](http://rethinkdb.com/api/python/db/) command.\n- `array_limit`: the maximum numbers of array elements that can be returned by a query (default: 100,000). This affects all ReQL commands that return arrays. Note that it has no effect on the size of arrays being _written_ to the database; those always have an upper limit of 100,000 elements.\n- `binary_format`: what format to return binary data in (default: `\'native\'`). Set this to `\'raw\'` if you want the raw pseudotype.\n- `min_batch_rows`: minimum number of rows to wait for before batching a result set (default: 8). This is an integer.\n- `max_batch_rows`: maximum number of rows to wait for before batching a result set (default: unlimited). This is an integer.\n- `max_batch_bytes`: maximum number of bytes to wait for before batching a result set (default: 1024). This is an integer.\n- `max_batch_seconds`: maximum number of seconds to wait before batching a result set (default: 0.5). This is a float (not an integer) and may be specified to the microsecond.\n- `first_batch_scaledown_factor`: factor to scale the other parameters down by on the first batch (default: 4). For example, with this set to 8 and `max_batch_rows` set to 80, on the first batch `max_batch_rows` will be adjusted to 10 (80 / 8). This allows the first batch to return faster.\n\n*Example* Run a query on the connection `conn` and print out every\nrow in the result.\n\n for doc in r.table(\'marvel\').run(conn):\n print doc\n\n*Example* If you are OK with potentially out of date data from all\nthe tables involved in this query and want potentially faster reads,\npass a flag allowing out of date data in an options object. Settings\nfor individual tables will supercede this global setting for all\ntables in the query.\n\n r.table(\'marvel\').run(conn, use_outdated=True)\n\n*Example* If you just want to send a write and forget about it, you\ncan set `noreply` to true in the options. In this case `run` will\nreturn immediately.\n\n r.table(\'marvel\').run(conn, noreply=True)\n\n*Example* If you want to specify whether to wait for a write to be\nwritten to disk (overriding the table\'s default settings), you can set\n`durability` to `\'hard\'` or `\'soft\'` in the options.\n\n r.table(\'marvel\')\n .insert({ \'superhero\': \'Iron Man\', \'superpower\': \'Arc Reactor\' })\n .run(conn, noreply=True, durability=\'soft\')\n\n*Example* If you do not want a time object to be converted to a\nnative date object, you can pass a `time_format` flag to prevent it\n(valid flags are "raw" and "native"). This query returns an object\nwith two fields (`epoch_time` and `$reql_type$`) instead of a native date\nobject.\n\n r.now().run(conn, time_format="raw")\n\n*Example* Specify the database to use for the query.\n\n for doc in r.table(\'marvel\').run(conn, db=\'heroes\'):\n print doc\n\nThis is equivalent to using the `db` command to specify the database:\n\n r.db(\'heroes\').table(\'marvel\').run(conn) ...\n\n*Example* Change the batching parameters for this query.\n\n r.table(\'marvel\').run(conn, max_batch_rows=16, max_batch_bytes=2048)\n'), + (rethinkdb.set_loop_type, b'r.set_loop_type(string)\n\nSet an asynchronous event loop model. Currently, the only event loop model RethinkDB supports is `"tornado"`, for use with the [Tornado web framework](http://www.tornadoweb.org). After setting the event loop to `"tornado"`, the [connect](http://rethinkdb.com/api/python/connect) and [run](http://rethinkdb.com/api/python/run) commands will return Tornado `Future` objects.\n\n*Example* Read a table\'s data using Tornado.\n\n r.set_loop_type("tornado")\n conn = r.connect(host=\'localhost\', port=28015)\n \n @gen.coroutine\n def use_cursor(conn):\n # Print every row in the table.\n cursor = yield r.table(\'test\').order_by(index="id").run(yield conn)\n while (yield cursor.fetch_next()):\n item = yield cursor.next()\n print(item)\n\nFor a longer discussion with Tornado examples, see the documentation article on [Asynchronous connections][ac].\n\n[ac]: /docs/async-connections/\n'), + (rethinkdb.net.Connection.use, b"conn.use(db_name)\n\nChange the default database on this connection.\n\n*Example* Change the default database so that we don't need to\nspecify the database when referencing a table.\n\n conn.use('marvel')\n r.table('heroes').run(conn) # refers to r.db('marvel').table('heroes')\n"), + (rethinkdb.ast.Table.config, b'table.config() -> selection<object>\ndatabase.config() -> selection<object>\n\nQuery (read and/or update) the configurations for individual tables or databases.\n\nThe `config` command is a shorthand way to access the `table_config` or `db_config` [System tables](http://rethinkdb.com/docs/system-tables/). It will return the single row from the system that corresponds to the database or table configuration, as if [get](http://rethinkdb.com/api/python/get) had been called on the system table with the UUID of the database or table in question.\n\n*Example* Get the configuration for the `users` table.\n\n r.table(\'users\').config().run(conn)\n \n {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "users",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "a", "replicas": ["a", "b"]},\n {"primary_replica": "d", "replicas": ["c", "d"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n\n*Example* Change the write acknowledgement requirement of the `users` table.\n\n r.table(\'users\').config().update({\'write_acks\': \'single\'}).run(conn)\n'), + (rethinkdb.ast.Table.rebalance, b'table.rebalance() -> object\ndatabase.rebalance() -> object\n\nRebalances the shards of a table. When called on a database, all the tables in that database will be rebalanced.\n\nThe `rebalance` command operates by measuring the distribution of primary keys within a table and picking split points that will give each shard approximately the same number of documents. It won\'t change the number of shards within a table, or change any other configuration aspect for the table or the database.\n\nA table will lose availability temporarily after `rebalance` is called; use the [wait](http://rethinkdb.com/api/python/wait) command to wait for the table to become available again, or [status](http://rethinkdb.com/api/python/status) to check if the table is available for writing.\n\nRethinkDB automatically rebalances tables when the number of shards are increased, and as long as your documents have evenly distributed primary keys—such as the default UUIDs—it is rarely necessary to call `rebalance` manually. Cases where `rebalance` may need to be called include:\n\n* Tables with unevenly distributed primary keys, such as incrementing integers\n* Changing a table\'s primary key type\n* Increasing the number of shards on an empty table, then using non-UUID primary keys in that table\n\nThe [web UI](http://rethinkdb.com/docs/administration-tools/) (and the [info](http://rethinkdb.com/api/python/info) command) can be used to tell you when a table\'s shards need to be rebalanced.\n\nThe return value of `rebalance` is an object with two fields:\n\n* `rebalanced`: the number of tables rebalanced.\n* `status_changes`: a list of new and old table status values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `rebalance` was executed. \n * `new_val`: The table\'s `status` value after `rebalance` was executed. (This value will almost always indicate the table is unavailable.)\n\nSee the [status](http://rethinkdb.com/api/python/status) command for an explanation of the objects returned in the `old_val` and `new_val` fields.\n\n*Example* Rebalance a table.\n\n r.table(\'superheroes\').rebalance().run(conn)\n \n {\n "rebalanced": 1,\n "status_changes": [\n {\n "old_val": {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": True,\n "ready_for_outdated_reads": True,\n "ready_for_reads": True,\n "ready_for_writes": True\n }\n },\n "new_val": {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "transitioning"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "transitioning"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": False,\n "ready_for_outdated_reads": False,\n "ready_for_reads": False,\n "ready_for_writes": False\n }\n }\n \n }\n ]\n }\n'), + (rethinkdb.ast.Table.reconfigure, b'table.reconfigure(shards=, replicas=[, primary_replica_tag=, dry_run=False]) -> object\ndatabase.reconfigure(shards=, replicas=[, primary_replica_tag=, dry_run=False]) -> object\n\nReconfigure a table\'s sharding and replication.\n\n* `shards`: the number of shards, an integer from 1-32. Required.\n* `replicas`: either an integer or a mapping object. Required.\n * If `replicas` is an integer, it specifies the number of replicas per shard. Specifying more replicas than there are servers will return an error.\n * If `replicas` is an object, it specifies key-value pairs of server tags and the number of replicas to assign to those servers: `{"tag1": 2, "tag2": 4, "tag3": 2, ...}`. For more information about server tags, read [Administration tools](http://rethinkdb.com/docs/administration-tools/).\n* `primary_replica_tag`: the primary server specified by its server tag. Required if `replicas` is an object; the tag must be in the object. This must *not* be specified if `replicas` is an integer.\n* `dry_run`: if `True` the generated configuration will not be applied to the table, only returned.\n\nThe return value of `reconfigure` is an object with three fields:\n\n* `reconfigured`: the number of tables reconfigured. This will be `0` if `dry_run` is `True`.\n* `config_changes`: a list of new and old table configuration values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [config](http://rethinkdb.com/api/python/config) value before `reconfigure` was executed. \n * `new_val`: The table\'s `config` value after `reconfigure` was executed.\n* `status_changes`: a list of new and old table status values. Each element of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `reconfigure` was executed. \n * `new_val`: The table\'s `status` value after `reconfigure` was executed.\n\nFor `config_changes` and `status_changes`, see the [config](http://rethinkdb.com/api/python/config) and [status](http://rethinkdb.com/api/python/status) commands for an explanation of the objects returned in the `old_val` and `new_val` fields.\n\nA table will lose availability temporarily after `reconfigure` is called; use the [table_status](http://rethinkdb.com/api/python/table_status) command to determine when the table is available again.\n\n**Note:** Whenever you call `reconfigure`, the write durability will be set to `hard` and the write acknowledgments will be set to `majority`; these can be changed by using the `config` command on the table.\n\nIf `reconfigure` is called on a database, all the tables in the database will have their configurations affected. The return value will be an array of the objects described above, one per table.\n\nRead [Sharding and replication](http://rethinkdb.com/docs/sharding-and-replication/) for a complete discussion of the subject, including advanced topics.\n\n*Example* Reconfigure a table.\n\n r.table(\'superheroes\').reconfigure(shards=2, replicas=1).run(conn)\n \n {\n "reconfigured": 1,\n "config_changes": [\n {\n "new_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "jeeves", "replicas": ["jeeves"]},\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n },\n "old_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n }\n ],\n "status_changes": [\n {\n "new_val": (status object),\n "old_val": (status object)\n }\n ]\n }\n\n*Example* Reconfigure a table, specifying replicas by server tags.\n\n r.table(\'superheroes\').reconfigure(shards=2, replicas={\'wooster\': 1, \'wayne\': 1}, primary_replica_tag=\'wooster\').run(conn)\n \n {\n "reconfigured": 1,\n "config_changes": [\n {\n "new_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "jeeves", "replicas": ["jeeves", "alfred"]},\n {"primary_replica": "jeeves", "replicas": ["jeeves", "alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n },\n "old_val": {\n "id": "31c92680-f70c-4a4b-a49e-b238eb12c023",\n "name": "superheroes",\n "db": "superstuff",\n "primary_key": "id",\n "shards": [\n {"primary_replica": "alfred", "replicas": ["alfred"]}\n ],\n "write_acks": "majority",\n "durability": "hard"\n }\n }\n ],\n "status_changes": [\n {\n "new_val": (status object),\n "old_val": (status object)\n }\n ]\n }\n'), + (rethinkdb.ast.Table.status, b'table.status() -> selection<object>\n\nReturn the status of a table.\n\nThe return value is an object providing information about the table\'s shards, replicas and replica readiness states. For a more complete discussion of the object fields, read about the `table_status` table in [System tables](http://rethinkdb.com/docs/system-tables/).\n\n* `db`: database name.\n* `name`: table name.\n* `id`: table UUID.\n* `shards`: an array of objects, one for each shard, with the following keys per object:\n * `primary_replica`: name of the shard\'s primary server.\n * `replicas`: an array of objects showing the status of each replica, with the following keys:\n * `server`: name of the replica server.\n * `state`: one of `ready`, `disconnected`, `backfilling_data`, `offloading_data`, `erasing_data`, `looking_for_primary_replica` or `transitioning`.\n* `status`: an object with the following boolean keys:\n * `all_replicas_ready`: `True` if all backfills have finished.\n * `ready_for_outdated_reads`: `True` if the table is ready for read queries with the `use_outdated` flag set to `True`.\n * `ready_for_reads`: `True` if the table is ready for read queries with current data (with the `use_outdated` flag set to `False` or unspecified).\n * `ready_for_writes`: `True` if the table is ready for write queries.\n\n*Example* Get a table\'s status.\n\n r.table(\'superheroes\').status().run(conn)\n \n {\n "db": "database",\n "id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n "name": "superheroes",\n "shards": [\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n },\n {\n "primary_replica": "jeeves",\n "replicas": [\n {\n "server": "jeeves",\n "state": "ready"\n }\n ]\n }\n ],\n "status": {\n "all_replicas_ready": True,\n "ready_for_outdated_reads": True,\n "ready_for_reads": True,\n "ready_for_writes": True\n }\n }\n'), + (rethinkdb.ast.Table.wait, b'table.wait([wait_for=\'ready_for_writes\', timeout=]) -> object\ndatabase.wait([wait_for=\'ready_for_writes\', timeout=]) -> object\nr.wait([wait_for=\'ready_for_writes\', timeout=]) -> object\n\nWait for a table or all the tables in a database to be ready. A table may be temporarily unavailable after creation, rebalancing or reconfiguring. The `wait` command blocks until the given table (or database) is fully up to date.\n\nThe `wait` command takes two optional arguments:\n\n* `wait_for`: a string indicating a table [status](http://rethinkdb.com/api/python/status) to wait on before returning, one of `ready_for_outdated_reads`, `ready_for_reads`, `ready_for_writes`, or `all_replicas_ready`. The default is `ready_for_writes`. \n* `timeout`: a number indicating maximum time to wait for in seconds before returning. The default is no timeout.\n\nThe return value is an object consisting of two key/value pairs:\n\n* `ready`: an integer indicating the number of tables waited for. It will always be `1` when `wait` is called on a table, and the total number of tables when called on a database.\n* `status_changes`: a list with one entry for each of the tables. Each member of the list will be an object with two fields:\n * `old_val`: The table\'s [status](http://rethinkdb.com/api/python/status) value before `wait` was executed. \n * `new_val`: The table\'s `status` value after `wait` finished.\n\nSee [status](http://rethinkdb.com/api/python/status) and [System tables](http://rethinkdb.com/docs/system-tables/) for a description of the fields within `status_changes`.\n\nIf `wait` is called with no table or database specified (the `r.wait()` form), it will wait on all the tables in the default database (set with the [connect](http://rethinkdb.com/api/python/connect/) command\'s `db` parameter, which defaults to `test`).\n\n*Example* Wait on a table to be ready.\n\n r.table(\'superheroes\').wait().run(conn)\n \n {\n "ready": 1,\n "status_changes": [\n \t{\n \t "old_val": {\n \t\t"db": "database",\n \t\t"id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n \t\t"name": "superheroes",\n \t\t"shards": [\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t },\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t }\n \t\t],\n \t\t"status": {\n \t\t "all_replicas_ready": True,\n \t\t "ready_for_outdated_reads": True,\n \t\t "ready_for_reads": True,\n \t\t "ready_for_writes": True\n \t\t}\n \t },\n \t "new_val": {\n \t\t"db": "database",\n \t\t"id": "5cb35225-81b2-4cec-9eef-bfad15481265",\n \t\t"name": "superheroes",\n \t\t"shards": [\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t },\n \t\t {\n \t\t\t"primary_replica": None,\n \t\t\t"replicas": [\n \t\t\t {\n \t\t\t\t"server": "jeeves",\n \t\t\t\t"state": "ready"\n \t\t\t }\n \t\t\t]\n \t\t }\n \t\t],\n \t\t"status": {\n \t\t "all_replicas_ready": True,\n \t\t "ready_for_outdated_reads": True,\n \t\t "ready_for_reads": True,\n \t\t "ready_for_writes": True\n \t\t}\n \t }\n \t}\n ]\n }\n'), + (rethinkdb.ast.RqlQuery.avg, b"sequence.avg([field_or_function]) -> number\n\nAverages all the elements of a sequence. If called with a field name,\naverages all the values of that field in the sequence, skipping\nelements of the sequence that lack that field. If called with a\nfunction, calls that function on every element of the sequence and\naverages the results, skipping elements of the sequence where that\nfunction returns `None` or a non-existence error.\n\nProduces a non-existence error when called on an empty sequence. You\ncan handle this case with `default`.\n\n*Example* What's the average of 3, 5, and 7?\n\n r.expr([3, 5, 7]).avg().run(conn)\n\n*Example* What's the average number of points scored in a game?\n\n r.table('games').avg('points').run(conn)\n\n*Example* What's the average number of points scored in a game,\ncounting bonus points?\n\n r.table('games').avg(lambda game:\n game['points'] + game['bonus_points']\n ).run(conn)\n\n*Example* What's the average number of points scored in a game?\n(But return `None` instead of raising an error if there are no games where\npoints have been scored.)\n\n r.table('games').avg('points').default(None).run(conn)\n"), + (rethinkdb.ast.RqlQuery.contains, b"sequence.contains(value|predicate[, value|predicate, ...]) -> bool\n\nWhen called with values, returns `True` if a sequence contains all the\nspecified values. When called with predicate functions, returns `True`\nif for each predicate there exists at least one element of the stream\nwhere that predicate returns `True`.\n\nValues and predicates may be mixed freely in the argument list.\n\n*Example* Has Iron Man ever fought Superman?\n\n r.table('marvel').get('ironman')['opponents'].contains('superman').run(conn)\n\n*Example* Has Iron Man ever defeated Superman in battle?\n\n r.table('marvel').get('ironman')['battles'].contains(lambda battle:\n (battle['winner'] == 'ironman') & (battle['loser'] == 'superman')\n ).run(conn)\n\n*Example* Use `contains` with a predicate function to simulate an `or`. Return the Marvel superheroes who live in Detroit, Chicago or Hoboken.\n\n r.table('marvel').filter(\n lambda hero: r.expr(['Detroit', 'Chicago', 'Hoboken']).contains(hero['city'])\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.count, b"sequence.count([value_or_predicate]) -> number\nbinary.count() -> number\n\nCounts the number of elements in a sequence. If called with a value,\ncounts the number of times that value occurs in the sequence. If\ncalled with a predicate function, counts the number of elements in the\nsequence where that function returns `True`.\n\nIf `count` is called on a [binary](http://rethinkdb.com/api/python/binary) object, it will return the size of the object in bytes.\n\n*Example* Count the number of users.\n\n r.table('users').count().run(conn)\n\n*Example* Count the number of 18 year old users.\n\n r.table('users')['age'].count(18).run(conn)\n\n*Example* Count the number of users over 18.\n\n r.table('users')['age'].count(lambda age: age > 18).run(conn)\n\n r.table('users').count(lambda user: user['age'] > 18).run(conn)\n"), + (rethinkdb.ast.RqlQuery.distinct, b"sequence.distinct() -> array\ntable.distinct([index=]) -> stream\n\nRemoves duplicate elements from a sequence.\n\nThe `distinct` command can be called on any sequence or table with an index.\n\n{% infobox %}\nWhile `distinct` can be called on a table without an index, the only effect will be to convert the table into a stream; the content of the stream will not be affected.\n{% endinfobox %}\n\n*Example* Which unique villains have been vanquished by Marvel heroes?\n\n r.table('marvel').concat_map(\n lambda hero: hero['villain_list']).distinct().run(conn)\n\n*Example* Topics in a table of messages have a secondary index on them, and more than one message can have the same topic. What are the unique topics in the table?\n\n r.table('messages').distinct(index='topics').run(conn)\n\nThe above structure is functionally identical to:\n\n r.table('messages')['topics'].distinct().run(conn)\n\nHowever, the first form (passing the index as an argument to `distinct`) is faster, and won't run into array limit issues since it's returning a stream.\n"), + (rethinkdb.ast.RqlQuery.group, b'sequence.group(field_or_function..., [index=\'index_name\', multi=False]) -> grouped_stream\n\nTakes a stream and partitions it into multiple groups based on the\nfields or functions provided.\n\nWith the `multi` flag single documents can be assigned to multiple groups, similar to the behavior of [multi-indexes](http://rethinkdb.com/docs/secondary-indexes/python). When `multi` is `True` and the grouping value is an array, documents will be placed in each group that corresponds to the elements of the array. If the array is empty the row will be ignored.\n\n*Example* Grouping games by player.\n\nSuppose that the table `games` has the following data:\n\n [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ]\n\nGrouping games by player can be done with:\n\n > r.table(\'games\').group(\'player\').run(conn)\n \n {\n "Alice": [\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ],\n "Bob": [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n\nCommands chained after `group` will be called on each of these grouped\nsub-streams, producing grouped data.\n\n*Example* What is each player\'s best game?\n\n > r.table(\'games\').group(\'player\').max(\'points\').run(conn)\n \n {\n "Alice": {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n "Bob": {"id": 2, "player": "Bob", "points": 15, "type": "ranked"}\n }\n\nCommands chained onto grouped data will operate on each grouped datum,\nproducing more grouped data.\n\n*Example* What is the maximum number of points scored by each player?\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].run(conn)\n \n {\n "Alice": 7,\n "Bob": 15\n }\n\nYou can also group by more than one field.\n\n*Example* What is the maximum number of points scored by each\nplayer for each game type?\n\n > r.table(\'games\').group(\'player\', \'type\').max(\'points\')[\'points\'].run(conn)\n \n {\n ("Alice", "free"): 7,\n ("Bob", "free"): 10,\n ("Bob", "ranked"): 15\n }\n\nYou can also group by a function.\n\n*Example* What is the maximum number of points scored by each\nplayer for each game type?\n\n > r.table(\'games\')\n .group(lambda game:\n game.pluck(\'player\', \'type\')\n ).max(\'points\')[\'points\'].run(conn)\n \n {\n frozenset([(\'player\', \'Alice\'), (\'type\', \'free\')]): 7,\n frozenset([(\'player\', \'Bob\'), (\'type\', \'free\')]): 10,\n frozenset([(\'player\', \'Bob\'), (\'type\', \'ranked\')]): 15,\n }\n\nUsing a function, you can also group by date on a ReQL [date field](http://rethinkdb.com/docs/dates-and-times/javascript/).\n\n*Example* How many matches have been played this year by month?\n\n > r.table(\'matches\').group(\n lambda match: [match[\'date\'].year(), match[\'date\'].month()]\n ).count().run(conn)\n \n {\n (2014, 2): 2,\n (2014, 3): 2,\n (2014, 4): 1,\n (2014, 5): 3\n }\n\nYou can also group on an index (primary key or secondary).\n\n*Example* What is the maximum number of points scored by game type?\n\n > r.table(\'games\').group(index=\'type\').max(\'points\')[\'points\'].run(conn)\n \n {\n "free": 10,\n "ranked": 15\n }\n\nSuppose that the table `games2` has the following data:\n\n [\n { \'id\': 1, \'matches\': {\'a\': [1, 2, 3], \'b\': [4, 5, 6]} },\n { \'id\': 2, \'matches\': {\'b\': [100], \'c\': [7, 8, 9]} },\n { \'id\': 3, \'matches\': {\'a\': [10, 20], \'c\': [70, 80]} }\n ]\n\nUsing the `multi` option we can group data by match A, B or C.\n\n > r.table(\'games2\').group(r.row[\'matches\'].keys(), multi=True).run(conn)\n \n [\n {\n \'group\': \'a\',\n \'reduction\': [ , ]\n },\n {\n \'group\': \'b\',\n \'reduction\': [ , ]\n },\n {\n \'group\': \'c\',\n \'reduction\': [ , ]\n }\n ]\n\n(The full result set is abbreviated in the figure; `, ` and `` would be the entire documents matching those keys.)\n\n*Example* Use [map](http://rethinkdb.com/api/python/map) and [sum](http://rethinkdb.com/api/python/sum) to get the total points scored for each match.\n\n r.table(\'games2\').group(r.row[\'matches\'].keys(), multi=True).ungroup().map(\n lambda doc: { \'match\': doc[\'group\'], \'total\': doc[\'reduction\'].sum(\n lambda set: set[\'matches\'][doc[\'group\']].sum()\n )}).run(conn)\n \n [\n { \'match\': \'a\', \'total\': 36 },\n { \'match\': \'b\', \'total\': 115 },\n { \'match\': \'c\', \'total\': 174 }\n ]\n\nThe inner `sum` adds the scores by match within each document; the outer `sum` adds those results together for a total across all the documents.\n\nIf you want to operate on all the groups rather than operating on each\ngroup (e.g. if you want to order the groups by their reduction), you\ncan use [ungroup](http://rethinkdb.com/api/python/ungroup/) to turn a grouped stream or\ngrouped data into an array of objects representing the groups.\n\n*Example* Ungrouping grouped data.\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].ungroup().run(conn)\n \n [\n {\n "group": "Alice",\n "reduction": 7\n },\n {\n "group": "Bob",\n "reduction": 15\n }\n ]\n\nUngrouping is useful e.g. for ordering grouped data, or for inserting\ngrouped data into a table.\n\n*Example* What is the maximum number of points scored by each\nplayer, with the highest scorers first?\n\n > r.table(\'games\').group(\'player\').max(\'points\')[\'points\'].ungroup().order_by(\n r.desc(\'reduction\')).run(conn)\n \n [\n {\n "group": "Bob",\n "reduction": 15\n },\n {\n "group": "Alice",\n "reduction": 7\n }\n ]\n\nWhen grouped data are returned to the client, they are transformed\ninto a client-specific native type. (Something similar is done with\n[times](http://rethinkdb.com/docs/dates-and-times/).) In Python, grouped data are\ntransformed into a `dictionary`. If the group value is an `array`, the\nkey is converted to a `tuple`. If the group value is a `dictionary`,\nit will be converted to a `frozenset`.\n\nIf you instead want to receive the raw\npseudotype from the server (e.g. if you\'re planning to serialize the\nresult as JSON), you can specify `group_format: \'raw\'` as an optional\nargument to `run`:\n\n*Example* Get back the raw `GROUPED_DATA` pseudotype.\n\n > r.table(\'games\').group(\'player\').avg(\'points\').run(conn, group_format=\'raw\')\n \n {\n "$reql_type$": "GROUPED_DATA",\n "data": [\n ["Alice", 4.5],\n ["Bob", 12.5]\n ]\n }\n\nNot passing the `group_format` flag would return:\n\n {\n "Alice": 4.5,\n "Bob": 12.5\n }\n\nYou might also want to use the [ungroup](http://rethinkdb.com/api/python/ungroup/)\ncommand (see above), which will turn the grouped data into an array of\nobjects on the server.\n\nIf you run a query that returns a grouped stream, it will be\nautomatically converted to grouped data before being sent back to you\n(there is currently no efficient way to stream groups from RethinkDB).\nThis grouped data is subject to the array size limit (see [run](http://rethinkdb.com/api/python/run)).\n\nIn general, operations on grouped streams will be efficiently\ndistributed, and operations on grouped data won\'t be. You can figure\nout what you\'re working with by putting `type_of` on the end of your\nquery. Below are efficient and inefficient examples.\n\n*Example* Efficient operation.\n\n # r.table(\'games\').group(\'player\').type_of().run(conn)\n # Returns "GROUPED_STREAM"\n r.table(\'games\').group(\'player\').min(\'points\').run(conn) # EFFICIENT\n\n*Example* Inefficient operation.\n\n # r.table(\'games\').group(\'player\').order_by(\'score\').type_of().run(conn)\n # Returns "GROUPED_DATA"\n r.table(\'games\').group(\'player\').order_by(\'score\').nth(0).run(conn) # INEFFICIENT\n\nWhat does it mean to be inefficient here? When operating on grouped\ndata rather than a grouped stream, *all* of the data has to be\navailable on the node processing the query. This means that the\noperation will only use one server\'s resources, and will require\nmemory proportional to the size of the grouped data it\'s operating\non. (In the case of the `order_by` in the inefficient example, that\nmeans memory proportional **to the size of the table**.) The array\nlimit is also enforced for grouped data, so the `order_by` example\nwould fail for tables with more than 100,000 rows unless you used the `array_limit` option with `run`.\n\n*Example* What is the maximum number of points scored by each\nplayer in free games?\n\n > r.table(\'games\').filter(lambda game:\n game[\'type\'] = \'free\'\n ).group(\'player\').max(\'points\')[\'points\'].run(conn)\n \n {\n "Alice": 7,\n "Bob": 10\n }\n\n*Example* What is each player\'s highest even and odd score?\n\n > r.table(\'games\')\n .group(\'name\', lambda game:\n game[\'points\'] % 2\n ).max(\'points\')[\'points\'].run(conn)\n \n {\n ("Alice", 1): 7,\n ("Bob", 0): 10,\n ("Bob", 1): 15\n }\n'), + (rethinkdb.ast.RqlQuery.max, b"sequence.max(field_or_function) -> element\nsequence.max(index='index') -> element\n\nFinds the maximum element of a sequence. The `max` command can be called with:\n\n* a **field name**, to return the element of the sequence with the largest value in that field;\n* an **index** (the primary key or a secondary index), to return the element of the sequence with the largest value in that index;\n* a **function**, to apply the function to every element within the sequence and return the element which returns the largest value from the function, ignoring any elements where the function produces a non-existence error.\n\nFor more information on RethinkDB's sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nCalling `max` on an empty sequence will throw a non-existence error; this can be handled using the [default](http://rethinkdb.com/api/python/default/) command.\n\n*Example* Return the maximum value in the list `[3, 5, 7]`.\n\n r.expr([3, 5, 7]).max().run(conn)\n\n*Example* Return the user who has scored the most points.\n\n r.table('users').max('points').run(conn)\n\n*Example* The same as above, but using a secondary index on the `points` field.\n\n r.table('users').max(index='points').run(conn)\n\n*Example* Return the user who has scored the most points, adding in bonus points from a separate field using a function.\n\n r.table('users').max(lambda user:\n user['points'] + user['bonus_points']\n ).run(conn)\n\n*Example* Return the highest number of points any user has ever scored. This returns the value of that `points` field, not a document.\n\n r.table('users').max('points')['points'].run(conn)\n\n*Example* Return the user who has scored the most points, but add a default `None` return value to prevent an error if no user has ever scored points.\n\n r.table('users').max('points').default(None).run(conn)\n"), + (rethinkdb.ast.RqlQuery.min, b"sequence.min(field_or_function) -> element\nsequence.min(index='index') -> element\n\nFinds the minimum element of a sequence. The `min` command can be called with:\n\n* a **field name**, to return the element of the sequence with the smallest value in that field;\n* an **index** (the primary key or a secondary index), to return the element of the sequence with the smallest value in that index;\n* a **function**, to apply the function to every element within the sequence and return the element which returns the smallest value from the function, ignoring any elements where the function produces a non-existence error.\n\nFor more information on RethinkDB's sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nCalling `min` on an empty sequence will throw a non-existence error; this can be handled using the [default](http://rethinkdb.com/api/python/default/) command.\n\n*Example* Return the minimum value in the list `[3, 5, 7]`.\n\n r.expr([3, 5, 7]).min().run(conn)\n\n*Example* Return the user who has scored the fewest points.\n\n r.table('users').min('points').run(conn)\n\n*Example* The same as above, but using a secondary index on the `points` field.\n\n r.table('users').min(index='points').run(conn)\n\n*Example* Return the user who has scored the fewest points, adding in bonus points from a separate field using a function.\n\n r.table('users').min(lambda user:\n user['points'] + user['bonus_points']\n ).run(conn)\n\n*Example* Return the smallest number of points any user has ever scored. This returns the value of that `points` field, not a document.\n\n r.table('users').min('points')['points'].run(conn)\n\n*Example* Return the user who has scored the fewest points, but add a default `None` return value to prevent an error if no user has ever scored points.\n\n r.table('users').min('points').default(None).run(conn)\n"), + (rethinkdb.ast.RqlQuery.reduce, b'sequence.reduce(reduction_function) -> value\n\nProduce a single value from a sequence through repeated application of a reduction\nfunction. \nThe reduction function can be called on:\n\n- two elements of the sequence\n- one element of the sequence and one result of a previous reduction\n- two results of previous reductions\n\nThe reduction function can be called on the results of two previous reductions because the\n`reduce` command is distributed and parallelized across shards and CPU cores. A common\nmistaken when using the `reduce` command is to suppose that the reduction is executed\nfrom left to right. Read the [map-reduce in RethinkDB](http://rethinkdb.com/docs/map-reduce/) article to\nsee an example.\n\nIf the sequence is empty, the server will produce a `RqlRuntimeError` that can be\ncaught with `default`. \nIf the sequence has only one element, the first element will be returned.\n\n*Example* Return the number of documents in the table `posts`.\n\n r.table("posts").map(lambda doc: 1)\n .reduce(lambda left, right: left+right)\n .default(0).run(conn)\n\nA shorter way to execute this query is to use [count](http://rethinkdb.com/api/python/count).\n\n*Example* Suppose that each `post` has a field `comments` that is an array of\ncomments. \nReturn the number of comments for all posts.\n\n r.table("posts").map(lambda doc:\n doc["comments"].count()\n ).reduce(lambda left, right:\n left+right\n ).default(0).run(conn)\n\n*Example* Suppose that each `post` has a field `comments` that is an array of\ncomments. \nReturn the maximum number comments per post.\n\n r.table("posts").map(lambda doc:\n doc["comments"].count()\n ).reduce(lambda left, right:\n r.branch(\n left > right,\n left,\n right\n )\n ).default(0).run(conn)\n\nA shorter way to execute this query is to use [max](http://rethinkdb.com/api/python/max).\n'), + (rethinkdb.ast.RqlQuery.sum, b"sequence.sum([field_or_function]) -> number\n\nSums all the elements of a sequence. If called with a field name,\nsums all the values of that field in the sequence, skipping elements\nof the sequence that lack that field. If called with a function,\ncalls that function on every element of the sequence and sums the\nresults, skipping elements of the sequence where that function returns\n`None` or a non-existence error.\n\nReturns `0` when called on an empty sequence.\n\n*Example* What's 3 + 5 + 7?\n\n r.expr([3, 5, 7]).sum().run(conn)\n\n*Example* How many points have been scored across all games?\n\n r.table('games').sum('points').run(conn)\n\n*Example* How many points have been scored across all games,\ncounting bonus points?\n\n r.table('games').sum(lambda game:\n game['points'] + game['bonus_points']\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.ungroup, b'grouped_stream.ungroup() -> array\ngrouped_data.ungroup() -> array\n\nTakes a grouped stream or grouped data and turns it into an array of\nobjects representing the groups. Any commands chained after `ungroup`\nwill operate on this array, rather than operating on each group\nindividually. This is useful if you want to e.g. order the groups by\nthe value of their reduction.\n\nThe format of the array returned by `ungroup` is the same as the\ndefault native format of grouped data in the JavaScript driver and\ndata explorer.\n\n*Example* What is the maximum number of points scored by each\nplayer, with the highest scorers first?\n\nSuppose that the table `games` has the following data:\n\n [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 5, "player": "Alice", "points": 7, "type": "free"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"},\n {"id": 12, "player": "Alice", "points": 2, "type": "free"}\n ]\n\nWe can use this query:\n\n r.table(\'games\')\n .group(\'player\').max(\'points\')[\'points\']\n .ungroup().order_by(r.desc(\'reduction\')).run(conn)\n\nResult: \n\n [\n {\n "group": "Bob",\n "reduction": 15\n },\n {\n "group": "Alice",\n "reduction": 7\n }\n ]\n\n*Example* Select one random player and all their games.\n\n r.table(\'games\').group(\'player\').ungroup().sample(1).run(conn)\n\nResult:\n\n [\n {\n "group": "Bob",\n "reduction": [\n {"id": 2, "player": "Bob", "points": 15, "type": "ranked"},\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n ]\n\nNote that if you didn\'t call `ungroup`, you would instead select one\nrandom game from each player:\n\n r.table(\'games\').group(\'player\').sample(1).run(conn)\n\nResult:\n\n {\n "Alice": [\n {"id": 5, "player": "Alice", "points": 7, "type": "free"}\n ],\n "Bob": [\n {"id": 11, "player": "Bob", "points": 10, "type": "free"}\n ]\n }\n\n*Example* Types!\n\n r.table(\'games\').group(\'player\').type_of().run(conn) # Returns "GROUPED_STREAM"\n r.table(\'games\').group(\'player\').ungroup().type_of().run(conn) # Returns "ARRAY"\n r.table(\'games\').group(\'player\').avg(\'points\').run(conn) # Returns "GROUPED_DATA"\n r.table(\'games\').group(\'player\').avg(\'points\').ungroup().run(conn) #Returns "ARRAY"\n'), + (rethinkdb.args, b"r.args(array) -> special\n\n`r.args` is a special term that's used to splice an array of arguments\ninto another term. This is useful when you want to call a variadic\nterm such as `get_all` with a set of arguments produced at runtime.\n\nThis is analogous to unpacking argument lists in Python.\n\n*Example* Get Alice and Bob from the table `people`.\n\n r.table('people').get_all('Alice', 'Bob').run(conn)\n # or\n r.table('people').get_all(r.args(['Alice', 'Bob'])).run(conn)\n\n*Example* Get all of Alice's children from the table `people`.\n\n # r.table('people').get('Alice') returns {'id': 'Alice', 'children': ['Bob', 'Carol']}\n r.table('people').get_all(r.args(r.table('people').get('Alice')['children'])).run(conn)\n"), + (rethinkdb.binary, b'r.binary(data) -> binary\n\nEncapsulate binary data within a query.\n\nThe type of data `binary` accepts depends on the client language. In Python, it expects a parameter of `bytes` type. Using a `bytes` object within a query implies the use of `binary` and the ReQL driver will automatically perform the coercion (in Python 3 only).\n\nBinary objects returned to the client in JavaScript will also be of the `bytes` type. This can be changed with the `binary_format` option provided to [run](http://rethinkdb.com/api/python/run) to return "raw" objects.\n\nOnly a limited subset of ReQL commands may be chained after `binary`:\n\n* [coerce_to](http://rethinkdb.com/api/python/coerce_to/) can coerce `binary` objects to `string` types\n* [count](http://rethinkdb.com/api/python/count/) will return the number of bytes in the object\n* [slice](http://rethinkdb.com/api/python/slice/) will treat bytes like array indexes (i.e., `slice(10,20)` will return bytes 10–19)\n* [type_of](http://rethinkdb.com/api/python/type_of) returns `PTYPE`\n* [info](http://rethinkdb.com/api/python/info) will return information on a binary object.\n\n*Example* Save an avatar image to a existing user record.\n\n f = open(\'./default_avatar.png\', \'rb\')\n avatar_image = f.read()\n f.close()\n r.table(\'users\').get(100).update({\'avatar\': r.binary(avatar_image)}).run(conn)\n\n*Example* Get the size of an existing avatar image.\n\n r.table(\'users\').get(100)[\'avatar\'].count().run(conn)\n \n 14156\n\nRead more details about RethinkDB\'s binary object support: [Storing binary objects](http://rethinkdb.com/docs/storing-binary/).\n'), + (rethinkdb.branch, b'r.branch(test, true_branch, false_branch) -> any\n\nIf the `test` expression returns `False` or `None`, the `false_branch` will be evaluated.\nOtherwise, the `true_branch` will be evaluated.\n \nThe `branch` command is effectively an `if` renamed due to language constraints.\n\n*Example* Return heroes and superheroes.\n\n r.table(\'marvel\').map(\n r.branch(\n r.row[\'victories\'] > 100,\n r.row[\'name\'] + \' is a superhero\',\n r.row[\'name\'] + \' is a hero\'\n )\n ).run(conn)\n\nIf the documents in the table `marvel` are:\n\n [{\n "name": "Iron Man",\n "victories": 214\n },\n {\n "name": "Jubilee",\n "victories": 9\n }]\n\nThe results will be:\n\n [\n "Iron Man is a superhero",\n "Jubilee is a hero"\n ]\n'), + (rethinkdb.ast.RqlQuery.coerce_to, b"sequence.coerce_to('array') -> array\nvalue.coerce_to('string') -> string\nstring.coerce_to('number') -> number\narray.coerce_to('object') -> object\nobject.coerce_to('array') -> array\nbinary.coerce_to('string') -> string\nstring.coerce_to('binary') -> binary\n\nConvert a value of one type into another.\n\n* a sequence, selection or object can be coerced to an array\n* an array of key-value pairs can be coerced to an object\n* a string can be coerced to a number\n* any datum (single value) can be coerced to a string\n* a binary object can be coerced to a string and vice-versa\n\n*Example* Coerce a stream to an array to store its output in a field. (A stream cannot be stored in a field directly.)\n\n r.table('posts').map(lambda post: post.merge(\n { 'comments': r.table('comments').get_all(post['id'], index='post_id').coerce_to('array') }\n )).run(conn)\n\n*Example* Coerce an array of pairs into an object.\n\n r.expr([['name', 'Ironman'], ['victories', 2000]]).coerce_to('object').run(conn)\n\n__Note:__ To coerce a list of key-value pairs like `['name', 'Ironman', 'victories', 2000]` to an object, use the [object](http://rethinkdb.com/api/python/object) command.\n\n*Example* Coerce a number to a string.\n\n r.expr(1).coerce_to('string').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.default, b'value.default(default_value) -> any\nsequence.default(default_value) -> any\n\nHandle non-existence errors. Tries to evaluate and return its first argument. If an\nerror related to the absence of a value is thrown in the process, or if its first\nargument returns `None`, returns its second argument. (Alternatively, the second argument\nmay be a function which will be called with either the text of the non-existence error\nor `None`.)\n\n*Example* Suppose we want to retrieve the titles and authors of the table `posts`.\nIn the case where the author field is missing or `None`, we want to retrieve the string\n`Anonymous`.\n\n r.table("posts").map(lambda post:\n {\n "title": post["title"],\n "author": post["author"].default("Anonymous")\n }\n ).run(conn)\n\nWe can rewrite the previous query with `r.branch` too.\n\n r.table("posts").map(lambda post:\n r.branch(\n post.has_fields("author"),\n {\n "title": post["title"],\n "author": post["author"]\n },\n {\n "title": post["title"],\n "author": "Anonymous" \n }\n )\n ).run(conn)\n\n*Example* The `default` command can be useful to filter documents too. Suppose\nwe want to retrieve all our users who are not grown-ups or whose age is unknown\n(i.e the field `age` is missing or equals `None`). We can do it with this query:\n\n r.table("users").filter(lambda user:\n (user["age"] < 18).default(True)\n ).run(conn)\n\nOne more way to write the previous query is to set the age to be `-1` when the\nfield is missing.\n\n r.table("users").filter(lambda user:\n user["age"].default(-1) < 18\n ).run(conn)\n\nOne last way to do the same query is to use `has_fields`.\n\n r.table("users").filter(lambda user:\n user.has_fields("age").not_() | (user["age"] < 18)\n ).run(conn)\n\nThe body of every `filter` is wrapped in an implicit `.default(False)`. You can overwrite\nthe value `False` by passing an option in filter, so the previous query can also be\nwritten like this.\n\n r.table("users").filter(\n lambda user: (user["age"] < 18).default(True),\n default=True\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.do, b"any.do(function) -> any\nr.do([args]*, function) -> any\nany.do(expr) -> any\nr.do([args]*, expr) -> any\n\nCall an anonymous function using return values from other ReQL commands or queries as arguments.\n\nThe last argument to `do` (or, in some forms, the only argument) is an expression or an anonymous function which receives values from either the previous arguments or from prefixed commands chained before `do`. The `do` command is essentially a single-element [map](http://rethinkdb.com/api/python/map/), letting you map a function over just one document. This allows you to bind a query result to a local variable within the scope of `do`, letting you compute the result just once and reuse it in a complex expression or in a series of ReQL commands.\n\nArguments passed to the `do` function must be basic data types, and cannot be streams or selections. (Read about [ReQL data types](http://rethinkdb.com/docs/data-types/).) While the arguments will all be evaluated before the function is executed, they may be evaluated in any order, so their values should not be dependent on one another. The type of `do`'s result is the type of the value returned from the function or last expression.\n\n*Example* Compute a golfer's net score for a game.\n\n r.table('players').get('86be93eb-a112-48f5-a829-15b2cb49de1d').do(\n lambda player: player['gross_score'] - player['course_handicap']\n ).run(conn)\n\n*Example* Return the name of the best scoring player in a two-player golf match.\n\n r.do(r.table('players').get(id1), r.table('players').get(id2),\n (lambda player1, player2:\n r.branch(player1['gross_score'].lt(player2['gross_score']),\n player1, player2))\n ).run(conn)\n\nNote that `branch`, the ReQL conditional command, must be used instead of `if`. See the `branch` [documentation](http://rethinkdb.com/api/python/branch) for more.\n\n*Example* Take different actions based on the result of a ReQL [insert](http://rethinkdb.com/api/python/insert) command.\n\n new_data = {\n 'id': 100,\n 'name': 'Agatha',\n 'gross_score': 57,\n 'course_handicap': 4\n }\n r.table('players').insert(new_data).do(lambda doc:\n r.branch((doc['inserted'] != 0),\n r.table('log').insert({'time': r.now(), 'response': doc, 'result': 'ok'}),\n r.table('log').insert({'time': r.now(), 'response': doc, 'result': 'error'}))\n ).run(conn)\n"), + (rethinkdb.error, b"r.error(message) -> error\n\nThrow a runtime error. If called with no arguments inside the second argument to `default`, re-throw the current error.\n\n*Example* Iron Man can't possibly have lost a battle:\n\n r.table('marvel').get('IronMan').do(\n lambda ironman: r.branch(ironman['victories'] < ironman['battles'],\n r.error('impossible code path'),\n ironman)\n ).run(conn)\n\n"), + (rethinkdb.expr, b"r.expr(value) -> value\n\nConstruct a ReQL JSON object from a native object.\n\nIf the native object is of the `bytes` type, then `expr` will return a binary object. See [binary](http://rethinkdb.com/api/python/binary) for more information.\n\n*Example* Objects wrapped with expr can then be manipulated by ReQL API functions.\n\n r.expr({'a':'b'}).merge({'b':[1,2,3]}).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.for_each, b"sequence.for_each(write_query) -> object\n\nLoop over a sequence, evaluating the given write query for each element.\n\n*Example* Now that our heroes have defeated their villains, we can safely remove them from the villain table.\n\n r.table('marvel').for_each(\n lambda hero: r.table('villains').get(hero['villainDefeated']).delete()\n ).run(conn)\n\n"), + (rethinkdb.http, b'r.http(url[, options]) -> value\nr.http(url[, options]) -> stream\n\nRetrieve data from the specified URL over HTTP. The return type depends on the `result_format` option, which checks the `Content-Type` of the response by default.\n\n*Example* Perform an HTTP `GET` and store the result in a table.\n\n r.table(\'posts\').insert(r.http(\'http://httpbin.org/get\')).run(conn)\n\nSee [the tutorial](http://rethinkdb.com/docs/external-api-access/) on `r.http` for more examples on how to use this command.\n\n* `timeout`: Number of seconds to wait before timing out and aborting the operation. Default: 30.\n\n* `reattempts`: An integer giving the number of attempts to make in cast of connection errors or potentially-temporary HTTP errors. Default: 5.\n\n* `redirects`: An integer giving the number of redirects and location headers to follow. Default: 1.\n\n* `verify`: Verify the server\'s SSL certificate, specified as a boolean. Default: True.\n\n* `result_format`: The format the result should be returned in. The values can be `\'text\'` (always return as a string), `\'json\'` (parse the result as JSON, raising an error if the parsing fails), `\'jsonp\'` (parse the result as [padded JSON](http://www.json-p.org/)), `\'binary\'` (return a binary object), or `\'auto\'` . The default is `\'auto\'`.\n\n When `result_format` is `\'auto\'`, the response body will be parsed according to the `Content-Type` of the response:\n * `application/json`: parse as `\'json\'`\n * `application/json-p`, `text/json-p`, `text/javascript`: parse as `\'jsonp\'`\n * `audio/*`, `video/*`, `image/*`, `application/octet-stream`: return a binary object\n * Anything else: parse as `\'text\'`\n\n* `method`: HTTP method to use for the request. One of `GET`, `POST`, `PUT`, `PATCH`, `DELETE` or `HEAD`. Default: `GET`.\n\n* `auth`: Authentication information in the form of an object with key/value pairs indicating the authentication type (in the `type` key) and any required information. Types currently supported are `basic` and `digest` for HTTP Basic and HTTP Digest authentication respectively. If `type` is omitted, `basic` is assumed. Example:\n\n\t```py\n\tr.http(\'http://httpbin.org/basic-auth/fred/mxyzptlk\',\n auth={ \'type\': \'basic\', \'user\': \'fred\', \'pass\': \'mxyzptlk\' }).run(conn)\n\t```\n\n* `params`: URL parameters to append to the URL as encoded key/value pairs, specified as an object. For example, `{ \'query\': \'banana\', \'limit\': 2 }` will be appended as `?query=banana&limit=2`. Default: none.\n\n* `header`: Extra header lines to include. The value may be an array of strings or an object. Default: none.\n\n Unless specified otherwise, `r.http` will by default use the headers `Accept-Encoding: deflate=1;gzip=0.5` and `User-Agent: RethinkDB/VERSION`.\n\n* `data`: Data to send to the server on a `POST`, `PUT`, `PATCH`, or `DELETE` request.\n\n For `PUT`, `PATCH` and `DELETE` requests, the value will be serialized to JSON and placed in the request body, and the `Content-Type` will be set to `application/json`.\n\n\tFor `POST` requests, data may be either an object or a string. Objects will be written to the body as form-encoded key/value pairs (values must be numbers, strings, or `None`). Strings will be put directly into the body. If `data` is not a string or an object, an error will be thrown.\n\n If `data` is not specified, no data will be sent.\n\n`r.http` supports depagination, which will request multiple pages in a row and aggregate the results into a stream. The use of this feature is controlled by the optional arguments `page` and `page_limit`. Either none or both of these arguments must be provided.\n\n* `page`: This option may specify either a built-in pagination strategy (as a string), or a function to provide the next URL and/or `params` to request.\n\n At the moment, the only supported built-in is `\'link-next\'`, which is equivalent to `lambda info: info[\'header\'][\'link\'][\'rel="next"\'].default(None)`.\n\n *Example* Perform a GitHub search and collect up to 3 pages of results.\n\n ```py\n r.http("https://api.github.com/search/code?q=addClass+user:mozilla",\n page=\'link-next\', page_limit=3).run(conn)\n ```\n\n As a function, `page` takes one parameter, an object of the format:\n\n ```py\n {\n \'params\': object, # the URL parameters used in the last request\n \'header\': object, # the HTTP headers of the last response as key/value pairs\n \'body\': value # the body of the last response in the format specified by `result_format`\n }\n ```\n\n The `header` field will be a parsed version of the header with fields lowercased, like so:\n\n ```py\n {\n \'content-length\': \'1024\',\n \'content-type\': \'application/json\',\n \'date\': \'Thu, 1 Jan 1970 00:00:00 GMT\',\n \'link\': {\n \'rel="last"\': \'http://example.com/?page=34\',\n \'rel="next"\': \'http://example.com/?page=2\'\n }\n }\n ```\n\n The `page` function may return a string corresponding to the next URL to request, `None` indicating that there is no more to get, or an object of the format:\n\n ```py\n {\n \'url\': string, # the next URL to request, or None for no more pages\n \'params\': object # new URL parameters to use, will be merged with the previous request\'s params\n }\n ```\n\n* `page_limit`: An integer specifying the maximum number of requests to issue using the `page` functionality. This is to prevent overuse of API quotas, and must be specified with `page`.\n * `-1`: no limit\n * `0`: no requests will be made, an empty stream will be returned\n * `n`: `n` requests will be made\n\n# Examples\n\n*Example* Perform multiple requests with different parameters.\n\n r.expr([1, 2, 3]).map(lambda i: r.http(\'http://httpbin.org/get\',\n params={ \'user\': i })).run(conn)\n\n*Example* Perform a `PUT` request for each item in a table.\n\n r.table(\'data\').map(lambda row: r.http(\'http://httpbin.org/put\',\n method=\'PUT\', data=row)).run(conn)\n\n*Example* Perform a `POST` request with accompanying data.\n\nUsing form-encoded data:\n\n r.http(\'http://httpbin.org/post\',\n method=\'POST\',\n data={ \'player\': \'Bob\', \'game\': \'tic tac toe\' }).run(conn)\n\nUsing JSON data:\n\n r.http(\'http://httpbin.org/post\',\n method=\'POST\',\n data=r.expr(value).coerce_to(\'string\'),\n header={ \'Content-Type\': \'application/json\' }).run(conn)\n\n*Example* Perform depagination with a custom `page` function.\n\n r.http(\'example.com/pages\',\n page=lambda info: info[\'body\'][\'meta\'][\'next\'].default(None),\n page_limit=5).run(conn)\n\n# Learn more\n\nSee [the tutorial](http://rethinkdb.com/docs/external-api-access/) on `r.http` for more examples on how to use this command.\n'), + (rethinkdb.ast.RqlQuery.info, b"any.info() -> object\n\nGet information about a ReQL value.\n\n*Example* Get information about a table such as primary key, or cache size.\n\n r.table('marvel').info().run(conn)\n\n"), + (rethinkdb.js, b'r.js(js_string[, timeout=]) -> value\n\nCreate a javascript expression.\n\n*Example* Concatenate two strings using JavaScript.\n\n`timeout` is the number of seconds before `r.js` times out. The default value is 5 seconds.\n\n{% infobox %}\nWhenever possible, you should use native ReQL commands rather than `r.js` for better performance.\n{% endinfobox %}\n\n r.js("\'str1\' + \'str2\'").run(conn)\n\n*Example* Select all documents where the \'magazines\' field is greater than 5 by running JavaScript on the server.\n\n r.table(\'marvel\').filter(\n r.js(\'(function (row) { return row.magazines.length > 5; })\')\n ).run(conn)\n\n*Example* You may also specify a timeout in seconds (defaults to 5).\n\n r.js(\'while(true) {}\', timeout=1.3).run(conn)\n\n'), + (rethinkdb.json, b'r.json(json_string) -> value\n\nParse a JSON string on the server.\n\n*Example* Send an array to the server\'\n\n r.json("[1,2,3]").run(conn)\n\n'), + (rethinkdb.range, b'r.range() -> stream\nr.range([start_value, ]end_value) -> stream\n\nGenerate a stream of sequential integers in a specified range. `range` takes 0, 1 or 2 arguments:\n\n* With no arguments, `range` returns an "infinite" stream from 0 up to and including the maximum integer value;\n* With one argument, `range` returns a stream from 0 up to but not including the end value;\n* With two arguments, `range` returns a stream from the start value up to but not including the end value.\n\nNote that the left bound (including the implied left bound of 0 in the 0- and 1-argument form) is always closed and the right bound is always open: the start value will always be included in the returned range and the end value will *not* be included in the returned range.\n\nAny specified arguments must be integers, or a `RqlRuntimeError` will be thrown. If the start value is equal or to higher than the end value, no error will be thrown but a zero-element stream will be returned.\n\n*Example* Return a four-element range of `[0, 1, 2, 3]`.\n\n > r.range(4).run(conn)\n \n [0, 1, 2, 3]\n\nYou can also use the [limit](http://rethinkdb.com/api/python/limit) command with the no-argument variant to achieve the same result in this case:\n\n > r.range().limit(4).run(conn)\n \n [0, 1, 2, 3]\n\n*Example* Return a range from -5 through 5.\n\n > r.range(-5, 6).run(conn)\n \n [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]\n'), + (rethinkdb.ast.RqlQuery.to_json_string, b'value.to_json_string() -> string\nvalue.to_json() -> string\n\nConvert a ReQL value or object to a JSON string. You may use either `to_json_string` or `to_json`.\n\n*Example* Get a ReQL document as a JSON string.\n\n > r.table(\'hero\').get(1).to_json()\n \n \'{"id": 1, "name": "Batman", "city": "Gotham", "powers": ["martial arts", "cinematic entrances"]}\'\n'), + (rethinkdb.ast.RqlQuery.to_json, b'value.to_json_string() -> string\nvalue.to_json() -> string\n\nConvert a ReQL value or object to a JSON string. You may use either `to_json_string` or `to_json`.\n\n*Example* Get a ReQL document as a JSON string.\n\n > r.table(\'hero\').get(1).to_json()\n \n \'{"id": 1, "name": "Batman", "city": "Gotham", "powers": ["martial arts", "cinematic entrances"]}\'\n'), + (rethinkdb.ast.RqlQuery.type_of, b'any.type_of() -> string\n\nGets the type of a value.\n\n*Example* Get the type of a string.\n\n r.expr("foo").type_of().run(conn)\n\n'), + (rethinkdb.uuid, b'r.uuid() -> string\n\nReturn a UUID (universally unique identifier), a string that can be used as a unique ID.\n\n*Example* Generate a UUID.\n\n > r.uuid().run(conn)\n \n 27961a0e-f4e8-4eb3-bf95-c5203e1d87b9\n'), + (rethinkdb.net.Cursor.close, b'cursor.close()\n\nClose a cursor. Closing a cursor cancels the corresponding query and frees the memory\nassociated with the open request.\n\n*Example* Close a cursor.\n\n cursor.close()\n'), + (rethinkdb.net.Cursor.next, b"cursor.next([wait=True])\n\nGet the next element in the cursor.\n\nThe optional `wait` argument specifies whether to wait for the next available element and how long to wait:\n\n* `True`: Wait indefinitely (the default).\n* `False`: Do not wait at all. If data is immediately available, it will be returned; if it is not available, a `RqlDriverError` will be raised.\n* number: Wait up the specified number of seconds for data to be available before raising `RqlDriverError`.\n\nThe behavior of `next` will be identical with `False`, `None` or the number `0`.\n\nCalling `next` the first time on a cursor provides the first element of the cursor. If the data set is exhausted (e.g., you have retrieved all the documents in a table), a `StopIteration` error will be raised when `next` is called.\n\n*Example* Retrieve the next element.\n\n cursor = r.table('superheroes').run(conn)\n doc = cursor.next()\n\n*Example* Retrieve the next element on a [changefeed](http://rethinkdb.com/docs/changefeeds/python), waiting up to five seconds.\n\n cursor = r.table('superheroes').changes().run(conn)\n doc = cursor.next(wait=5)\n\n__Note:__ RethinkDB sequences can be iterated through via the Python [Iterable][it] interface. The canonical way to retrieve all the results is to use a [for...in](../each/) loop or [list()](../to_array/).\n\n[it]: https://docs.python.org/3.4/library/stdtypes.html#iterator-types\n"), + (rethinkdb.ast.RqlQuery.date, b'time.date() -> time\n\nReturn a new time object only based on the day, month and year (ie. the same day at 00:00).\n\n*Example* Retrieve all the users whose birthday is today\n\n r.table("users").filter(lambda user:\n user["birthdate"].date() == r.now().date()\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.day, b'time.day() -> number\n\nReturn the day of a time object as a number between 1 and 31.\n\n*Example* Return the users born on the 24th of any month.\n\n r.table("users").filter(\n r.row["birthdate"].day() == 24\n )\n\n'), + (rethinkdb.ast.RqlQuery.day_of_week, b'time.day_of_week() -> number\n\nReturn the day of week of a time object as a number between 1 and 7 (following ISO 8601 standard). For your convenience, the terms r.monday, r.tuesday etc. are defined and map to the appropriate integer.\n\n*Example* Return today\'s day of week.\n\n r.now().day_of_week().run(conn)\n\n*Example* Retrieve all the users who were born on a Tuesday.\n\n r.table("users").filter( lambda user:\n user["birthdate"].day_of_week().eq(r.tuesday)\n )\n\n'), + (rethinkdb.ast.RqlQuery.day_of_year, b'time.day_of_year() -> number\n\nReturn the day of the year of a time object as a number between 1 and 366 (following ISO 8601 standard).\n\n*Example* Retrieve all the users who were born the first day of a year.\n\n r.table("users").filter(\n r.row["birthdate"].day_of_year() == 1\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.during, b'time.during(start_time, end_time[, left_bound="closed", right_bound="open"])\n -> bool\n\nReturn whether a time is between two other times. By default, this is inclusive of the start time and exclusive of the end time. Set `left_bound` and `right_bound` to explicitly include (`closed`) or exclude (`open`) that endpoint of the range.\n\n*Example* Retrieve all the posts that were posted between December 1st, 2013 (inclusive) and December 10th, 2013 (exclusive).\n\n r.table("posts").filter(\n r.row[\'date\'].during(r.time(2013, 12, 1, "Z"), r.time(2013, 12, 10, "Z"))\n ).run(conn)\n\n*Example* Retrieve all the posts that were posted between December 1st, 2013 (exclusive) and December 10th, 2013 (inclusive).\n\n r.table("posts").filter(\n r.row[\'date\'].during(r.time(2013, 12, 1, "Z"), r.time(2013, 12, 10, "Z"), left_bound="open", right_bound="closed")\n ).run(conn)\n\n'), + (rethinkdb.epoch_time, b'r.epoch_time(epoch_time) -> time\n\nCreate a time object based on seconds since epoch. The first argument is a double and\nwill be rounded to three decimal places (millisecond-precision).\n\n*Example* Update the birthdate of the user "John" to November 3rd, 1986.\n\n r.table("user").get("John").update({"birthdate": r.epoch_time(531360000)}).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.hours, b'time.hours() -> number\n\nReturn the hour in a time object as a number between 0 and 23.\n\n*Example* Return all the posts submitted after midnight and before 4am.\n\n r.table("posts").filter(lambda post:\n post["date"].hours() < 4\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.in_timezone, b"time.in_timezone(timezone) -> time\n\nReturn a new time object with a different timezone. While the time stays the same, the results returned by methods such as hours() will change since they take the timezone into account. The timezone argument has to be of the ISO 8601 format.\n\n*Example* Hour of the day in San Francisco (UTC/GMT -8, without daylight saving time).\n\n r.now().in_timezone('-08:00').hours().run(conn)\n"), + (rethinkdb.iso8601, b'r.iso8601(iso8601Date[, default_timezone=\'\']) -> time\n\nCreate a time object based on an ISO 8601 date-time string (e.g. \'2013-01-01T01:01:01+00:00\'). We support all valid ISO 8601 formats except for week dates. If you pass an ISO 8601 date-time without a time zone, you must specify the time zone with the `default_timezone` argument. Read more about the ISO 8601 format at [Wikipedia](http://en.wikipedia.org/wiki/ISO_8601).\n\n*Example* Update the time of John\'s birth.\n\n r.table("user").get("John").update({"birth": r.iso8601(\'1986-11-03T08:30:00-07:00\')}).run(conn)\n'), + (rethinkdb.ast.RqlQuery.minutes, b'time.minutes() -> number\n\nReturn the minute in a time object as a number between 0 and 59.\n\n*Example* Return all the posts submitted during the first 10 minutes of every hour.\n\n r.table("posts").filter(lambda post:\n post["date"].minutes() < 10\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.month, b'time.month() -> number\n\nReturn the month of a time object as a number between 1 and 12. For your convenience, the terms r.january, r.february etc. are defined and map to the appropriate integer.\n\n*Example* Retrieve all the users who were born in November.\n\n r.table("users").filter(\n r.row["birthdate"].month() == 11\n )\n\n*Example* Retrieve all the users who were born in November.\n\n r.table("users").filter(\n r.row["birthdate"].month() == r.november\n )\n\n'), + (rethinkdb.now, b'r.now() -> time\n\nReturn a time object representing the current time in UTC. The command now() is computed once when the server receives the query, so multiple instances of r.now() will always return the same time inside a query.\n\n*Example* Add a new user with the time at which he subscribed.\n\n r.table("users").insert({\n "name": "John",\n "subscription_date": r.now()\n }).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.seconds, b'time.seconds() -> number\n\nReturn the seconds in a time object as a number between 0 and 59.999 (double precision).\n\n*Example* Return the post submitted during the first 30 seconds of every minute.\n\n r.table("posts").filter(lambda post:\n post["date"].seconds() < 30\n ).run(conn)\n\n'), + (rethinkdb.time, b'r.time(year, month, day[, hour, minute, second], timezone)\n -> time\n\nCreate a time object for a specific time.\n\nA few restrictions exist on the arguments:\n\n- `year` is an integer between 1400 and 9,999.\n- `month` is an integer between 1 and 12.\n- `day` is an integer between 1 and 31.\n- `hour` is an integer.\n- `minutes` is an integer.\n- `seconds` is a double. Its value will be rounded to three decimal places\n(millisecond-precision).\n- `timezone` can be `\'Z\'` (for UTC) or a string with the format `\xc2\xb1[hh]:[mm]`.\n\n*Example* Update the birthdate of the user "John" to November 3rd, 1986 UTC.\n\n r.table("user").get("John").update({"birthdate": r.time(1986, 11, 3, \'Z\')}).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.time_of_day, b'time.time_of_day() -> number\n\nReturn the number of seconds elapsed since the beginning of the day stored in the time object.\n\n*Example* Retrieve posts that were submitted before noon.\n\n r.table("posts").filter(\n r.row["date"].time_of_day() <= 12*60*60\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.timezone, b'time.timezone() -> string\n\nReturn the timezone of the time object.\n\n*Example* Return all the users in the "-07:00" timezone.\n\n r.table("users").filter(lambda user:\n user["subscriptionDate"].timezone() == "-07:00"\n )\n\n'), + (rethinkdb.ast.RqlQuery.to_epoch_time, b'time.to_epoch_time() -> number\n\nConvert a time object to its epoch time.\n\n*Example* Return the current time in seconds since the Unix Epoch with millisecond-precision.\n\n r.now().to_epoch_time()\n\n'), + (rethinkdb.ast.RqlQuery.to_iso8601, b'time.to_iso8601() -> string\n\nConvert a time object to a string in ISO 8601 format.\n\n*Example* Return the current ISO 8601 time.\n\n > r.now().to_iso8601().run(conn)\n \n "2015-04-20T18:37:52.690+00:00"\n\n'), + (rethinkdb.ast.RqlQuery.year, b'time.year() -> number\n\nReturn the year of a time object.\n\n*Example* Retrieve all the users born in 1986.\n\n r.table("users").filter(lambda user:\n user["birthdate"].year() == 1986\n ).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.append, b"array.append(value) -> array\n\nAppend a value to an array.\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].append('newBoots').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.__getitem__, b"sequence[attr] -> sequence\nsingleSelection[attr] -> value\nobject[attr] -> value\narray[index] -> value\n\nGet a single field from an object. If called on a sequence, gets that field from every object in the sequence, skipping objects that lack it.\n\n*Example* What was Iron Man's first appearance in a comic?\n\n r.table('marvel').get('IronMan')['firstAppearance'].run(conn)\n\nThe `[]` command also accepts integer arguments as array offsets, like the [nth](http://rethinkdb.com/api/python/nth) command.\n\n*Example* Get the fourth element in a sequence. (The first element is position `0`, so the fourth element is position `3`.)\n\n r.expr([10, 20, 30, 40, 50])[3]\n \n 40\n"), + (rethinkdb.ast.RqlQuery.change_at, b'array.change_at(index, value) -> array\n\nChange a value in an array at a given index. Returns the modified array.\n\n*Example* Bruce Banner hulks out.\n\n r.expr(["Iron Man", "Bruce", "Spider-Man"]).change_at(1, "Hulk").run(conn)\n'), + (rethinkdb.ast.RqlQuery.delete_at, b"array.delete_at(index [,endIndex]) -> array\n\nRemove one or more elements from an array at a given index. Returns the modified array. (Note: `delete_at` operates on arrays, not documents; to delete documents, see the [delete](http://rethinkdb.com/api/python/delete) command.)\n\nIf only `index` is specified, `delete_at` removes the element at that index. If both `index` and `end_index` are specified, `delete_at` removes the range of elements between `index` and `end_index`, inclusive of `index` but not inclusive of `end_index`.\n\nIf `end_index` is specified, it must not be less than `index`. Both `index` and `end_index` must be within the array's bounds (i.e., if the array has 10 elements, an `index` or `end_index` of 10 or higher is invalid).\n\nBy using a negative `index` you can delete from the end of the array. `-1` is the last element in the array, `-2` is the second-to-last element, and so on. You may specify a negative `end_index`, although just as with a positive value, this will not be inclusive. The range `(2,-1)` specifies the third element through the next-to-last element.\n\n*Example* Delete the second element of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(1).run(conn)\n \n ['a', 'c', 'd', 'e', 'f']\n\n*Example* Delete the second and third elements of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(1,3).run(conn)\n \n ['a', 'd', 'e', 'f']\n\n*Example* Delete the next-to-last element of an array.\n\n > r.expr(['a','b','c','d','e','f']).delete_at(-2).run(conn)\n \n ['a', 'b', 'c', 'd', 'f']\n\n*Example* Delete a comment on a post.\n\nGiven a post document such as:\n\n{\n id: '4cf47834-b6f9-438f-9dec-74087e84eb63',\n title: 'Post title',\n author: 'Bob',\n comments: [\n { author: 'Agatha', text: 'Comment 1' },\n { author: 'Fred', text: 'Comment 2' }\n ]\n}\n\nThe second comment can be deleted by using `update` and `delete_at` together.\n\n r.table('posts').get('4cf47834-b6f9-438f-9dec-74087e84eb63').update(\n lambda post: { 'comments': post['comments'].delete_at(1) }\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.difference, b"array.difference(array) -> array\n\nRemove the elements of one array from another array.\n\n*Example* Retrieve Iron Man's equipment list without boots.\n\n r.table('marvel').get('IronMan')['equipment'].difference(['Boots']).run(conn)\n\n*Example* Remove Iron Man's boots from his equipment.\n\n r.table('marvel').get('IronMan')[:equipment].update(lambda doc:\n {'equipment': doc['equipment'].difference(['Boots'])}\n ).run(conn)\n"), + (rethinkdb.ast.RqlQuery.get_field, b"sequence.get_field(attr) -> sequence\nsingleSelection.get_field(attr) -> value\nobject.get_field(attr) -> value\n\nGet a single field from an object. If called on a sequence, gets that field from every\nobject in the sequence, skipping objects that lack it.\n\n*Example* What was Iron Man's first appearance in a comic?\n\n r.table('marvel').get('IronMan').get_field('firstAppearance').run(conn)\n"), + (rethinkdb.ast.RqlQuery.has_fields, b'sequence.has_fields([selector1, selector2...]) -> stream\narray.has_fields([selector1, selector2...]) -> array\nobject.has_fields([selector1, selector2...]) -> boolean\n\nTest if an object has one or more fields. An object has a field if it has that key and the key has a non-null value. For instance, the object `{\'a\': 1,\'b\': 2,\'c\': null}` has the fields `a` and `b`.\n\nWhen applied to a single object, `has_fields` returns `true` if the object has the fields and `false` if it does not. When applied to a sequence, it will return a new sequence (an array or stream) containing the elements that have the specified fields.\n\n*Example* Return the players who have won games.\n\n r.table(\'players\').has_fields(\'games_won\').run(conn)\n\n*Example* Return the players who have *not* won games. To do this, use `has_fields` with [not](http://rethinkdb.com/api/python/not), wrapped with [filter](http://rethinkdb.com/api/python/filter).\n\n r.table(\'players\').filter(~r.row.has_fields(\'games_won\')).run(conn)\n\n*Example* Test if a specific player has won any games.\n\n r.table(\'players\').get(\n \'b5ec9714-837e-400c-aa74-dbd35c9a7c4c\').has_fields(\'games_won\').run(conn)\n\n**Nested Fields**\n\n`has_fields` lets you test for nested fields in objects. If the value of a field is itself a set of key/value pairs, you can test for the presence of specific keys.\n\n*Example* In the `players` table, the `games_won` field contains one or more fields for kinds of games won:\n\n {\n \'games_won\': {\n \'playoffs\': 2,\n \'championships\': 1\n }\n }\n\nReturn players who have the "championships" field.\n\n r.table(\'players\').has_fields({\'games_won\': {\'championships\': true}}).run(conn)\n\nNote that `true` in the example above is testing for the existence of `championships` as a field, not testing to see if the value of the `championships` field is set to `true`. There\'s a more convenient shorthand form available. (See [pluck](http://rethinkdb.com/api/python/pluck) for more details on this.)\n\n r.table(\'players\').has_fields({\'games_won\': \'championships\'}).run(conn)\n'), + (rethinkdb.ast.RqlQuery.insert_at, b'array.insert_at(index, value) -> array\n\nInsert a value in to an array at a given index. Returns the modified array.\n\n*Example* Hulk decides to join the avengers.\n\n r.expr(["Iron Man", "Spider-Man"]).insert_at(1, "Hulk").run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.keys, b"singleSelection.keys() -> array\nobject.keys() -> array\n\nReturn an array containing all of the object's keys.\n\n*Example* Get all the keys of a row.\n\n r.table('marvel').get('ironman').keys().run(conn)\n\n"), + (rethinkdb.literal, b'r.literal(object) -> special\n\nReplace an object in a field instead of merging it with an existing object in a `merge` or `update` operation. = Using `literal` with no arguments in a `merge` or `update` operation will remove the corresponding field.\n\n*Example* Replace one nested document with another rather than merging the fields.\n\nAssume your users table has this structure:\n\n [\n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 18,\n "city": "Dallas"\n }\n } \n ...\n ]\n\nUsing `update` to modify the `data` field will normally merge the nested documents:\n\n r.table(\'users\').get(1).update({ \'data\': { \'age\': 19, \'job\': \'Engineer\' } }).run(conn)\n \n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 19,\n "city": "Dallas",\n "job": "Engineer"\n }\n } \n\nThat will preserve `city` and other existing fields. But to replace the entire `data` document with a new object, use `literal`:\n\n r.table(\'users\').get(1).update({ \'data\': r.literal({ \'age\': 19, \'job\': \'Engineer\' }) }).run(conn)\n \n {\n "id": 1,\n "name": "Alice",\n "data": {\n "age": 19,\n "job": "Engineer"\n }\n } \n\n*Example* Use `literal` to remove a field from a document.\n\n r.table(\'users\').get(1).merge({ "data": r.literal() }).run(conn)\n \n {\n "id": 1,\n "name": "Alice"\n }\n'), + (rethinkdb.ast.RqlQuery.merge, b'singleSelection.merge(object|function[, object|function, ...]) -> object\nobject.merge(object|function[, object|function, ...]) -> object\nsequence.merge(object|function[, object|function, ...]) -> stream\narray.merge(object|function[, object|function, ...]) -> array\n\nMerge two or more objects together to construct a new object with properties from all. When there is a conflict between field names, preference is given to fields in the rightmost object in the argument list `merge` also accepts a subquery function that returns an object, which will be used similarly to a [map](http://rethinkdb.com/api/python/map/) function.\n\n*Example* Equip Thor for battle.\n\n r.table(\'marvel\').get(\'thor\').merge(\n r.table(\'equipment\').get(\'hammer\'),\n r.table(\'equipment\').get(\'pimento_sandwich\')\n ).run(conn)\n\n*Example* Equip every hero for battle, using a subquery function to retrieve their weapons.\n\n r.table(\'marvel\').merge(lambda hero:\n { \'weapons\': r.table(\'weapons\').get(hero[\'weapon_id\']) }\n ).run(conn)\n\n*Example* Use `merge` to join each blog post with its comments.\n\nNote that the sequence being merged—in this example, the comments—must be coerced from a selection to an array. Without `coerce_to` the operation will throw an error ("Expected type DATUM but found SELECTION").\n\n r.table(\'posts\').merge(lambda post:\n { \'comments\': r.table(\'comments\').get_all(post[\'id\'],\n index=\'post_id\').coerce_to(\'array\') }\n ).run(conn)\n\n*Example* Merge can be used recursively to modify object within objects.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10}}}\n ).run(conn)\n\n*Example* To replace a nested object with another object you can use the literal keyword.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : r.literal({\'repulsor rays\' : {\'dmg\' : 3, \'cooldown\' : 0}})}\n ).run(conn)\n\n*Example* Literal can be used to remove keys from an object as well.\n\n r.expr({\'weapons\' : {\'spectacular graviton beam\' : {\'dmg\' : 10, \'cooldown\' : 20}}}).merge(\n {\'weapons\' : {\'spectacular graviton beam\' : r.literal()}}\n ).run(conn)\n\n'), + (rethinkdb.object, b'r.object([key, value,]...) -> object\n\nCreates an object from a list of key-value pairs, where the keys must\nbe strings. `r.object(A, B, C, D)` is equivalent to\n`r.expr([[A, B], [C, D]]).coerce_to(\'OBJECT\')`.\n\n*Example* Create a simple object.\n\n > r.object(\'id\', 5, \'data\', [\'foo\', \'bar\']).run(conn)\n {\'data\': ["foo", "bar"], \'id\': 5}\n'), + (rethinkdb.ast.RqlQuery.pluck, b"sequence.pluck([selector1, selector2...]) -> stream\narray.pluck([selector1, selector2...]) -> array\nobject.pluck([selector1, selector2...]) -> object\nsingleSelection.pluck([selector1, selector2...]) -> object\n\nPlucks out one or more attributes from either an object or a sequence of objects\n(projection).\n\n*Example* We just need information about IronMan's reactor and not the rest of the\ndocument.\n\n r.table('marvel').get('IronMan').pluck('reactorState', 'reactorPower').run(conn)\n\n*Example* For the hero beauty contest we only care about certain qualities.\n\n r.table('marvel').pluck('beauty', 'muscleTone', 'charm').run(conn)\n\n*Example* Pluck can also be used on nested objects.\n\n r.table('marvel').pluck({'abilities' : {'damage' : True, 'mana_cost' : True}, 'weapons' : True}).run(conn)\n\n*Example* The nested syntax can quickly become overly verbose so there's a shorthand\nfor it.\n\n r.table('marvel').pluck({'abilities' : ['damage', 'mana_cost']}, 'weapons').run(conn)\n\nFor more information read the [nested field documentation](http://rethinkdb.com/docs/nested-fields/).\n"), + (rethinkdb.ast.RqlQuery.prepend, b"array.prepend(value) -> array\n\nPrepend a value to an array.\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].prepend('newBoots').run(conn)\n"), + (rethinkdb.row, b"r.row -> value\n\nReturns the currently visited document. Note that `row` does not work within subqueries to access nested documents; you should use anonymous functions to access those documents instead. (See the last example.)\n\n*Example* Get all users whose age is greater than 5.\n\n r.table('users').filter(r.row['age'] > 5).run(conn)\n\n*Example* Access the attribute 'child' of an embedded document.\n\n r.table('users').filter(r.row['embedded_doc']['child'] > 5).run(conn)\n\n*Example* Add 1 to every element of an array.\n\n r.expr([1, 2, 3]).map(r.row + 1).run(conn)\n\n*Example* For nested queries, use functions instead of `row`.\n\n r.table('users').filter(\n lambda doc: doc['name'] == r.table('prizes').get('winner')\n ).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.set_difference, b"array.set_difference(array) -> array\n\nRemove the elements of one array from another and return them as a set (an array with\ndistinct values).\n\n*Example* Check which pieces of equipment Iron Man has, excluding a fixed list.\n\n r.table('marvel').get('IronMan')['equipment'].set_difference(['newBoots', 'arc_reactor']).run(conn)\n"), + (rethinkdb.ast.RqlQuery.set_insert, b"array.set_insert(value) -> array\n\nAdd a value to an array and return it as a set (an array with distinct values).\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots.\n\n r.table('marvel').get('IronMan')['equipment'].set_insert('newBoots').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.set_intersection, b"array.set_intersection(array) -> array\n\nIntersect two arrays returning values that occur in both of them as a set (an array with\ndistinct values).\n\n*Example* Check which pieces of equipment Iron Man has from a fixed list.\n\n r.table('marvel').get('IronMan')['equipment'].set_intersection(['newBoots', 'arc_reactor']).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.set_union, b"array.set_union(array) -> array\n\nAdd a several values to an array and return it as a set (an array with distinct values).\n\n*Example* Retrieve Iron Man's equipment list with the addition of some new boots and an arc reactor.\n\n r.table('marvel').get('IronMan')['equipment'].set_union(['newBoots', 'arc_reactor']).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.splice_at, b'array.splice_at(index, array) -> array\n\nInsert several values in to an array at a given index. Returns the modified array.\n\n*Example* Hulk and Thor decide to join the avengers.\n\n r.expr(["Iron Man", "Spider-Man"]).splice_at(1, ["Hulk", "Thor"]).run(conn)\n'), + (rethinkdb.ast.RqlQuery.without, b"sequence.without([selector1, selector2...]) -> stream\narray.without([selector1, selector2...]) -> array\nsingleSelection.without([selector1, selector2...]) -> object\nobject.without([selector1, selector2...]) -> object\n\nThe opposite of pluck; takes an object or a sequence of objects, and returns them with\nthe specified paths removed.\n\n*Example* Since we don't need it for this computation we'll save bandwidth and leave\nout the list of IronMan's romantic conquests.\n\n r.table('marvel').get('IronMan').without('personalVictoriesList').run(conn)\n\n*Example* Without their prized weapons, our enemies will quickly be vanquished.\n\n r.table('enemies').without('weapons').run(conn)\n\n*Example* Nested objects can be used to remove the damage subfield from the weapons and abilities fields.\n\n r.table('marvel').without({'weapons' : {'damage' : True}, 'abilities' : {'damage' : True}}).run(conn)\n\n*Example* The nested syntax can quickly become overly verbose so there's a shorthand for it.\n\n r.table('marvel').without({'weapons' : 'damage', 'abilities' : 'damage'}).run(conn)\n\n"), + (rethinkdb.circle, b"r.circle([longitude, latitude], radius[, num_vertices=32, geo_system='WGS84', unit='m', fill=True]) -> geometry\nr.circle(point, radius[, {num_vertices=32, geo_system='WGS84', unit='m', fill=True]) -> geometry\n\nConstruct a circular line or polygon. A circle in RethinkDB is a polygon or line *approximating* a circle of a given radius around a given center, consisting of a specified number of vertices (default 32).\n\nThe center may be specified either by two floating point numbers, the latitude (−90 to 90) and longitude (−180 to 180) of the point on a perfect sphere (see [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system), or by a point object. The radius is a floating point number whose units are meters by default, although that may be changed with the `unit` argument.\n\nOptional arguments available with `circle` are:\n\n* `num_vertices`: the number of vertices in the polygon or line. Defaults to 32.\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n* `unit`: Unit for the radius distance. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n* `fill`: if `True` (the default) the circle is filled, creating a polygon; if `False` the circle is unfilled (creating a line).\n\n*Example* Define a circle.\n\n r.table('geo').insert({\n 'id': 300,\n 'name': 'Hayes Valley',\n 'neighborhood': r.circle([-122.423246,37.779388], 1000)\n }).run(conn)\n"), + (rethinkdb.ast.RqlQuery.distance, b"geometry.distance(geometry[, geo_system='WGS84', unit='m']) -> number\n\nCompute the distance between a point and another geometry object. At least one of the geometry objects specified must be a point.\n\nOptional arguments available with `distance` are:\n\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n* `unit`: Unit to return the distance in. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n\nIf one of the objects is a polygon or a line, the point will be projected onto the line or polygon assuming a perfect sphere model before the distance is computed (using the model specified with `geo_system`). As a consequence, if the polygon or line is extremely large compared to Earth's radius and the distance is being computed with the default WGS84 model, the results of `distance` should be considered approximate due to the deviation between the ellipsoid and spherical models.\n\n*Example* Compute the distance between two points on the Earth in kilometers.\n\n > point1 = r.point(-122.423246,37.779388)\n > point2 = r.point(-117.220406,32.719464)\n > r.distance(point1, point2, unit='km').run(conn)\n \n 734.1252496021841\n"), + (rethinkdb.ast.RqlQuery.fill, b"line.fill() -> polygon\n\nConvert a Line object into a Polygon object. If the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\nIf the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them. You cannot directly construct a polygon with holes in it using `polygon`, but you can use [polygon_sub](http://rethinkdb.com/api/python/polygon_sub) to use a second polygon within the interior of the first to define a hole.\n\n*Example* Create a line object and then convert it to a polygon.\n\n r.table('geo').insert({\n 'id': 201,\n 'rectangle': r.line(\n [-122.423246,37.779388],\n [-122.423246,37.329898],\n [-121.886420,37.329898],\n [-121.886420,37.779388]\n )\n }).run(conn)\n \n r.table('geo').get(201).update({\n 'rectangle': r.row('rectangle').fill()\n }).run(conn)\n"), + (rethinkdb.geojson, b"r.geojson(geojson) -> geometry\n\nConvert a [GeoJSON][] object to a ReQL geometry object.\n\n[GeoJSON]: http://geojson.org\n\nRethinkDB only allows conversion of GeoJSON objects which have ReQL equivalents: Point, LineString, and Polygon. MultiPoint, MultiLineString, and MultiPolygon are not supported. (You could, however, store multiple points, lines and polygons in an array and use a geospatial multi index with them.)\n\nOnly longitude/latitude coordinates are supported. GeoJSON objects that use Cartesian coordinates, specify an altitude, or specify their own coordinate reference system will be rejected.\n\n*Example* Convert a GeoJSON object to a ReQL geometry object.\n\n geo_json = {\n 'type': 'Point',\n 'coordinates': [ -122.423246, 37.779388 ]\n }\n r.table('geo').insert({\n 'id': 'sfo',\n 'name': 'San Francisco',\n 'location': r.geojson(geo_json)\n }).run(conn)\n"), + (rethinkdb.ast.Table.get_intersecting, b"table.get_intersecting(geometry, index='indexname') -> selection\n\nGet all documents where the given geometry object intersects the geometry object of the requested geospatial index.\n\nThe `index` argument is mandatory. This command returns the same results as `table.filter(r.row('index').intersects(geometry))`. The total number of results is limited to the array size limit which defaults to 100,000, but can be changed with the `array_limit` option to [run](http://rethinkdb.com/api/python/run).\n\n*Example* Which of the locations in a list of parks intersect `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks').get_intersecting(circle1, index='area').run(conn)\n"), + (rethinkdb.ast.Table.get_nearest, b"table.get_nearest(point, index='indexname'[, max_results=100, max_dist=100000, unit='m', geo_system='WGS84']) -> array\n\nGet all documents where the specified geospatial index is within a certain distance of the specified point (default 100 kilometers).\n\nThe `index` argument is mandatory. Optional arguments are:\n\n* `max_results`: the maximum number of results to return (default 100).\n* `unit`: Unit for the distance. Possible values are `m` (meter, the default), `km` (kilometer), `mi` (international mile), `nm` (nautical mile), `ft` (international foot).\n* `max_dist`: the maximum distance from an object to the specified point (default 100 km).\n* `geo_system`: the reference ellipsoid to use for geographic coordinates. Possible values are `WGS84` (the default), a common standard for Earth's geometry, or `unit_sphere`, a perfect sphere of 1 meter radius.\n\nThe return value will be an array of two-item objects with the keys `dist` and `doc`, set to the distance between the specified point and the document (in the units specified with `unit`, defaulting to meters) and the document itself, respectively.\n\n*Example* Return a list of enemy hideouts within 5000 meters of the secret base.\n\n secret_base = r.point(-122.422876,37.777128)\n r.table('hideouts').get_nearest(secret_base, index='location',\n max_dist=5000).run(conn)\n"), + (rethinkdb.ast.RqlQuery.includes, b"sequence.includes(geometry) -> sequence\ngeometry.includes(geometry) -> bool\n\nTests whether a geometry object is completely contained within another. When applied to a sequence of geometry objects, `includes` acts as a [filter](http://rethinkdb.com/api/python/filter), returning a sequence of objects from the sequence that include the argument.\n\n*Example* Is `point2` included within a 2000-meter circle around `point1`?\n\n > point1 = r.point(-117.220406,32.719464)\n > point2 = r.point(-117.206201,32.725186)\n > r.circle(point1, 2000).includes(point2).run(conn)\n \n True\n\n*Example* Which of the locations in a list of parks include `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks')['area'].includes(circle1).run(conn)\n"), + (rethinkdb.ast.RqlQuery.intersects, b"sequence.intersects(geometry) -> sequence\ngeometry.intersects(geometry) -> bool\n\nTests whether two geometry objects intersect with one another. When applied to a sequence of geometry objects, `intersects` acts as a [filter](http://rethinkdb.com/api/python/filter), returning a sequence of objects from the sequence that intersect with the argument.\n\n*Example* Is `point2` within a 2000-meter circle around `point1`?\n\n > point1 = r.point(-117.220406,32.719464)\n > point2 = r.point(-117.206201,32.725186)\n > r.circle(point1, 2000).intersects(point2).run(conn)\n \n True\n\n*Example* Which of the locations in a list of parks intersect `circle1`?\n\n circle1 = r.circle([-117.220406,32.719464], 10, unit='mi')\n r.table('parks')('area').intersects(circle1).run(conn)\n"), + (rethinkdb.line, b"r.line([lon1, lat1], [lon2, lat2], ...) -> line\nr.line(point1, point2, ...) -> line\n\nConstruct a geometry object of type Line. The line can be specified in one of two ways:\n\n* Two or more two-item arrays, specifying latitude and longitude numbers of the line's vertices;\n* Two or more [Point](http://rethinkdb.com/api/python/point) objects specifying the line's vertices.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\n*Example* Define a line.\n\n r.table('geo').insert({\n 'id': 101,\n 'route': r.line([-122.423246,37.779388], [-121.886420,37.329898])\n }).run(conn)\n"), + (rethinkdb.point, b"r.point(longitude, latitude) -> point\n\nConstruct a geometry object of type Point. The point is specified by two floating point numbers, the longitude (−180 to 180) and latitude (−90 to 90) of the point on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\n*Example* Define a point.\n\n r.table('geo').insert({\n 'id': 1,\n 'name': 'San Francisco',\n 'location': r.point(-122.423246,37.779388)\n }).run(conn)\n"), + (rethinkdb.polygon, b"r.polygon([lon1, lat1], [lon2, lat2], ...) -> polygon\nr.polygon(point1, point2, ...) -> polygon\n\nConstruct a geometry object of type Polygon. The Polygon can be specified in one of two ways:\n\n* Three or more two-item arrays, specifying latitude and longitude numbers of the polygon's vertices;\n* Three or more [Point](http://rethinkdb.com/api/python/point) objects specifying the polygon's vertices.\n\nLongitude (−180 to 180) and latitude (−90 to 90) of vertices are plotted on a perfect sphere. See [Geospatial support](http://rethinkdb.com/docs/geo-support/) for more information on ReQL's coordinate system.\n\nIf the last point does not specify the same coordinates as the first point, `polygon` will close the polygon by connecting them. You cannot directly construct a polygon with holes in it using `polygon`, but you can use [polygon_sub](http://rethinkdb.com/api/python/polygon_sub) to use a second polygon within the interior of the first to define a hole.\n\n*Example* Define a polygon.\n\n r.table('geo').insert({\n 'id': 101,\n 'rectangle': r.polygon(\n [-122.423246,37.779388],\n [-122.423246,37.329898],\n [-121.886420,37.329898],\n [-121.886420,37.779388]\n )\n }).run(conn)\n"), + (rethinkdb.ast.RqlQuery.polygon_sub, b'polygon1.polygon_sub(polygon2) -> polygon\n\nUse `polygon2` to "punch out" a hole in `polygon1`. `polygon2` must be completely contained within `polygon1` and must have no holes itself (it must not be the output of `polygon_sub` itself).\n\n*Example* Define a polygon with a hole punched in it.\n\n outer_polygon = r.polygon(\n [-122.4,37.7],\n [-122.4,37.3],\n [-121.8,37.3],\n [-121.8,37.7]\n )\n inner_polygon = r.polygon(\n [-122.3,37.4],\n [-122.3,37.6],\n [-122.0,37.6],\n [-122.0,37.4]\n )\n outer_polygon.polygon_sub(inner_polygon).run(conn)\n'), + (rethinkdb.ast.RqlQuery.to_geojson, b"geometry.to_geojson() -> object\n\nConvert a ReQL geometry object to a [GeoJSON][] object.\n\n[GeoJSON]: http://geojson.org\n\n*Example* Convert a ReQL geometry object to a GeoJSON object.\n\n > r.table(geo).get('sfo')['location'].to_geojson().run(conn)\n \n {\n 'type': 'Point',\n 'coordinates': [ -122.423246, 37.779388 ]\n }\n"), + (rethinkdb.ast.RqlQuery.eq_join, b'sequence.eq_join(left_field, right_table[, index=\'id\']) -> sequence\n\nJoin tables using a field on the left-hand sequence matching primary keys or secondary indexes on the right-hand table. `eq_join` is more efficient than other Re_qL join types, and operates much faster. Documents in the result set consist of pairs of left-hand and right-hand documents, matched when the field on the left-hand side exists and is non-null and an entry with that field\'s value exists in the specified index on the right-hand side.\n\nThe result set of `eq_join` is a stream or array of objects. Each object in the returned set will be an object of the form `{ left: , right: }`, where the values of `left` and `right` will be the joined documents. Use the zip command to merge the `left` and `right` fields together.\n\n**Example:** Match players with the games they\'ve played against one another.\n\nThe players table contains these documents:\n\n [\n { \'id\': 1, \'player\': \'George\', \'gameId\': 1 },\n { \'id\': 2, \'player\': \'Agatha\', \'gameId\': 3 },\n { \'id\': 3, \'player\': \'Fred\', \'gameId\': 2 },\n { \'id\': 4, \'player\': \'Marie\', \'gameId\': 2 },\n { \'id\': 5, \'player\': \'Earnest\', \'gameId\': 1 },\n { \'id\': 6, \'player\': \'Beth\', \'gameId\': 3 }\n ]\n\nThe games table contains these documents:\n\n [\n { \'id\': 1, \'field\': \'Little Delving\' },\n { \'id\': 2, \'field\': \'Rushock Bog\' },\n { \'id\': 3, \'field\': \'Bucklebury\' }\n ]\n\nJoin these tables using `game_id` on the player table and `id` on the games table:\n\n r.table(\'players\').eq_join(\'game_id\', r.table(\'games\')).run(conn)\n\nThis will return a result set such as the following:\n\n [\n {\n "left" : { "gameId" : 3, "id" : 2, "player" : "Agatha" },\n "right" : { "id" : 3, "field" : "Bucklebury" }\n },\n {\n "left" : { "gameId" : 2, "id" : 3, "player" : "Fred" },\n "right" : { "id" : 2, "field" : "Rushock Bog" }\n },\n ...\n ]\n\nWhat you likely want is the result of using `zip` with that. For clarity, we\'ll use `without` to drop the `id` field from the games table (it conflicts with the `id` field for the players and it\'s redundant anyway), and we\'ll order it by the games.\n\n r.table(\'players\').eq_join(\'game_id\', r.table(\'games\')).without({\'right\': "id"}).zip().order_by(\'game_id\').run(conn)\n \n [\n { "field": "Little Delving", "gameId": 1, "id": 5, "player": "Earnest" },\n { "field": "Little Delving", "gameId": 1, "id": 1, "player": "George" },\n { "field": "Rushock Bog", "gameId": 2, "id": 3, "player": "Fred" },\n { "field": "Rushock Bog", "gameId": 2, "id": 4, "player": "Marie" },\n { "field": "Bucklebury", "gameId": 3, "id": 6, "player": "Beth" },\n { "field": "Bucklebury", "gameId": 3, "id": 2, "player": "Agatha" }\n ]\n\nFor more information, see [Table joins in Rethink_dB](http://rethinkdb.com/docs/table-joins/).\n\n**Example:** Use a secondary index on the right table rather than the primary key. If players have a secondary index on their cities, we can get a list of arenas with players in the same area.\n\n r.table(\'arenas\').eq_join(\'city_id\', r.table(\'arenas\'), index=\'city_id\').run(conn)\n\n**Example:** Use a nested key as the join field. Suppose the documents in the players table were structured like this:\n\n { \'id\': 1, \'player\': \'George\', \'game\': {\'id\': 1} },\n { \'id\': 2, \'player\': \'Agatha\', \'game\': {\'id\': 3} },\n ...\n\nSimply specify the field using the `row` command instead of a string.\n\n r.table(\'players\').eq_join(r.row[\'game\'][\'id\'], r.table(\'games\')).without({\'right\': \'id\'}).zip().run(conn)\n \n [\n { "field": "Little Delving", "game": { "id": 1 }, "id": 5, "player": "Earnest" },\n { "field": "Little Delving", "game": { "id": 1 }, "id": 1, "player": "George" },\n ...\n ]\n\n**Example:** Use a function instead of a field to join on a more complicated expression. Suppose the players have lists of favorite games ranked in order in a field such as `"favorites": [3, 2, 1]`. Get a list of players and their top favorite:\n\n r.table(\'players3\').eq_join(\n lambda player: player[\'favorites\'].nth(0),\n r.table(\'games\')\n ).without([{\'left\': [\'favorites\', \'game_id\', \'id\']}, {\'right\': \'id\'}]).zip()\n\nResult:\n\n [\n \t{ "field": "Rushock Bog", "name": "Fred" },\n \t{ "field": "Little Delving", "name": "George" },\n \t...\n ]\n'), + (rethinkdb.ast.RqlQuery.inner_join, b"sequence.inner_join(other_sequence, predicate) -> stream\narray.inner_join(other_sequence, predicate) -> array\n\nReturns an inner join of two sequences. The returned sequence represents an intersection of the left-hand sequence and the right-hand sequence: each row of the left-hand sequence will be compared with each row of the right-hand sequence to find all pairs of rows which satisfy the predicate. Each matched pair of rows of both sequences are combined into a result row. In most cases, you will want to follow the join with [zip](http://rethinkdb.com/api/python/zip) to combine the left and right results.\n\nNote that `inner_join` is slower and much less efficient than using [eq_join](http://rethinkdb.com/api/python/eq_join/) or [concat_map](http://rethinkdb.com/api/python/concat_map/) with [get_all](http://rethinkdb.com/api/python/get_all/). You should avoid using `inner_join` in commands when possible.\n\n*Example* Return a list of all matchups between Marvel and DC heroes in which the DC hero could beat the Marvel hero in a fight.\n\n r.table('marvel').inner_join(r.table('dc'),\n lambda marvel_row, dc_row: marvel_row['strength'] < dc_row['strength']\n ).zip().run(conn)\n\n(Compare this to an [outer_join](http://rethinkdb.com/api/python/outer_join) with the same inputs and predicate, which would return a list of *all* Marvel heroes along with any DC heroes with a higher strength.)"), + (rethinkdb.ast.RqlQuery.outer_join, b"sequence.outer_join(other_sequence, predicate) -> stream\narray.outer_join(other_sequence, predicate) -> array\n\nReturns a left outer join of two sequences. The returned sequence represents a union of the left-hand sequence and the right-hand sequence: all documents in the left-hand sequence will be returned, each matched with a document in the right-hand sequence if one satisfies the predicate condition. In most cases, you will want to follow the join with [zip](http://rethinkdb.com/api/python/zip) to combine the left and right results.\n\nNote that `outer_join` is slower and much less efficient than using [concat_map](http://rethinkdb.com/api/python/concat_map/) with [get_all](http://rethinkdb.com/api/python/get_all). You should avoid using `outer_join` in commands when possible.\n\n*Example* Return a list of all Marvel heroes, paired with any DC heroes who could beat them in a fight.\n\n r.table('marvel').outer_join(r.table('dc'),\n lambda marvel_row, dc_row: marvel_row['strength'] < dc_row['strength']\n ).zip().run(conn)\n\n(Compare this to an [inner_join](http://rethinkdb.com/api/python/inner_join) with the same inputs and predicate, which would return a list only of the matchups in which the DC hero has the higher strength.)\n"), + (rethinkdb.ast.RqlQuery.zip, b"stream.zip() -> stream\narray.zip() -> array\n\nUsed to 'zip' up the result of a join by merging the 'right' fields into 'left' fields of each member of the sequence.\n\n*Example* 'zips up' the sequence by merging the left and right fields produced by a join.\n\n r.table('marvel').eq_join('main_dc_collaborator', r.table('dc')).zip().run(conn)\n"), + (rethinkdb.db_create, b'r.db_create(db_name) -> object\n\nCreate a database. A RethinkDB database is a collection of tables, similar to\nrelational databases.\n\nIf successful, the command returns an object with two fields:\n\n* `dbs_created`: always `1`.\n* `config_changes`: a list containing one object with two fields, `old_val` and `new_val`:\n * `old_val`: always `None`.\n * `new_val`: the database\'s new [config](http://rethinkdb.com/api/python/config) value.\n\nIf a database with the same name already exists, the command throws `RqlRuntimeError`.\n\nNote: Only alphanumeric characters and underscores are valid for the database name.\n\n*Example* Create a database named \'superheroes\'.\n\n r.db_create(\'superheroes\').run(conn)\n \n {\n "config_changes": [\n {\n "new_val": {\n "id": "e4689cfc-e903-4532-a0e6-2d6797a43f07",\n "name": "superheroes"\n },\n "old_val": None\n }\n ],\n "dbs_created": 1\n }\n\n'), + (rethinkdb.db_drop, b'r.db_drop(db_name) -> object\n\nDrop a database. The database, all its tables, and corresponding data will be deleted.\n\nIf successful, the command returns an object with two fields:\n\n* `dbs_dropped`: always `1`.\n* `tables_dropped`: the number of tables in the dropped database.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: the database\'s original [config](http://rethinkdb.com/api/python/config) value.\n * `new_val`: always `None`.\n\nIf the given database does not exist, the command throws `RqlRuntimeError`.\n\n*Example* Drop a database named \'superheroes\'.\n\n r.db_drop(\'superheroes\').run(conn)\n \n {\n "config_changes": [\n {\n "old_val": {\n "id": "e4689cfc-e903-4532-a0e6-2d6797a43f07",\n "name": "superheroes"\n },\n "new_val": None\n }\n ],\n "tables_dropped": 3,\n "dbs_dropped": 1\n }\n\n'), + (rethinkdb.db_list, b'r.db_list() -> array\n\nList all database names in the system. The result is a list of strings.\n\n*Example* List all databases.\n\n r.db_list().run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.changes, b'stream.changes(squash=True, include_states=False) -> stream\nsingleSelection.changes(squash=True, include_states=False) -> stream\n\nReturn an infinite stream of objects representing changes to a query.\n\nThe `squash` optional argument controls how `changes` batches change notifications:\n\n* `True`: When multiple changes to the same document occur before a batch of notifications is sent, the changes are "squashed" into one change. The client receives a notification that will bring it fully up to date with the server. This is the default.\n* `False`: All changes will be sent to the client verbatim.\n* `n`: A numeric value (floating point). Similar to `True`, but the server will wait `n` seconds to respond in order to squash as many changes together as possible, reducing network traffic.\n\nIf the `include_states` optional argument is `True`, the changefeed stream will include special status documents consisting of the field `state` and a string indicating a change in the feed\'s state. These documents can occur at any point in the feed between the notification documents described below. There are currently two states:\n\n* `{"state": "initializing"}` indicates the following documents represent initial values on the feed rather than changes. This will be the first document of a feed that returns initial values.\n* `{"state": "ready"}` indicates the following documents represent changes. This will be the first document of a feed that does *not* return initial values; otherwise, it will indicate the initial values have all been sent.\n\nPoint changefeeds will always return initial values and have an `initializing` state; feeds that return changes on unfiltered tables will never return initial values. Feeds that return changes on more complex queries may or may not return return initial values, depending on the kind of aggregation. Read the article on [Changefeeds in RethinkDB][cfr] for a more detailed discussion. If `include_states` is `True` on a changefeed that does not return initial values, the first document on the feed will be `{"state": "ready"}`.\n\n[cfr]: /docs/changefeeds/python/\n\nIf `include_states` is `False` (the default), the status documents will not be sent on the feed.\n\nIf the table becomes unavailable, the changefeed will be disconnected, and a runtime exception will be thrown by the driver.\n\nChangefeed notifications take the form of a two-field object:\n\n {\n "old_val": ,\n "new_val": \n }\n\nThe first notification object in the changefeed stream will contain the query\'s initial value in `new_val` and have no `old_val` field. When a document is deleted, `new_val` will be `None`; when a document is inserted, `old_val` will be `None`.\n\nCertain document transformation commands can be chained before changefeeds. For more information, read the [discussion of changefeeds][cfr] in the "Query language" documentation.\n\nThe server will buffer up to 100,000 elements. If the buffer limit is hit, early changes will be discarded, and the client will receive an object of the form `{"error": "Changefeed cache over array size limit, skipped X elements."}` where `X` is the number of elements skipped.\n\nCommands that operate on streams (such as `filter` or `map`) can usually be chained after `changes`. However, since the stream produced by `changes` has no ending, commands that need to consume the entire stream before returning (such as `reduce` or `count`) cannot.\n\nIt\'s a good idea to open changefeeds on their own connection. If you don\'t, other queries run on the same connection will experience unpredictable latency spikes while the connection blocks on more changes.\n\n*Example* Subscribe to the changes on a table.\n\nStart monitoring the changefeed in one client:\n\n for change in r.table(\'games\').changes().run(conn):\n print change\n\nAs these queries are performed in a second client, the first client would receive and print the following objects:\n\n > r.table(\'games\').insert({\'id\': 1}).run(conn)\n {\'old_val\': None, \'new_val\': {\'id\': 1}}\n \n > r.table(\'games\').get(1).update({\'player1\': \'Bob\'}).run(conn)\n {\'old_val\': {\'id\': 1}, \'new_val\': {\'id\': 1, \'player1\': \'Bob\'}}\n \n > r.table(\'games\').get(1).replace({\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}).run(conn)\n {\'old_val\': {\'id\': 1, \'player1\': \'Bob\'},\n \'new_val\': {\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}}\n \n > r.table(\'games\').get(1).delete().run(conn)\n {\'old_val\': {\'id\': 1, \'player1\': \'Bob\', \'player2\': \'Alice\'}, \'new_val\': None}\n \n > r.table_drop(\'games\').run(conn)\n RqlRuntimeError: Changefeed aborted (table unavailable)\n\n*Example* Return all the changes that increase a player\'s score.\n\n r.table(\'test\').changes().filter(\n r.row[\'new_val\'][\'score\'] > r.row[\'old_val\'][\'score\']\n ).run(conn)\n\n*Example* Return all the changes to Bob\'s score.\n\n # Note that this will have to look at and discard all the changes to\n # rows besides Bob\'s. This is currently no way to filter with an index\n # on changefeeds.\n r.table(\'test\').changes().filter(r.row[\'new_val\'][\'name\'].eq(\'Bob\')).run(conn)\n\n*Example* Return all the inserts on a table.\n\n r.table(\'test\').changes().filter(r.row[\'old_val\'].eq(None)).run(conn)\n\n*Example* Return all the changes to game 1, with state notifications.\n\n r.table(\'games\').get(1).changes(include_states=True).run(conn)\n \n # result returned on changefeed\n {"state": "initializing"}\n {"new_val": {"id": 1, "score": 12, "arena": "Hobbiton Field"}}\n {"state": "ready"}\n {\n \t"old_val": {"id": 1, "score": 12, "arena": "Hobbiton Field"},\n \t"new_val": {"id": 1, "score": 14, "arena": "Hobbiton Field"}\n }\n {\n \t"old_val": {"id": 1, "score": 14, "arena": "Hobbiton Field"},\n \t"new_val": {"id": 1, "score": 17, "arena": "Hobbiton Field", "winner": "Frodo"}\n }\n\n*Example* Return all the changes to the top 10 games. This assumes the presence of a `score` secondary index on the `games` table.\n\n r.table(\'games\').order_by(index=r.desc(\'score\')).limit(10).run(conn)\n'), + (rethinkdb.ast.Table.index_create, b'table.index_create(index_name[, index_function][, multi=False, geo=False]) -> object\n\nCreate a new secondary index on a table. Secondary indexes improve the speed of many read queries at the slight cost of increased storage space and decreased write performance. For more information about secondary indexes, read the article "[Using secondary indexes in RethinkDB](http://rethinkdb.com/docs/secondary-indexes/)."\n\nRethinkDB supports different types of secondary indexes:\n\n- *Simple indexes* based on the value of a single field.\n- *Compound indexes* based on multiple fields.\n- *Multi indexes* based on arrays of values.\n- *Geospatial indexes* based on indexes of geometry objects, created when the `geo` optional argument is true.\n- Indexes based on *arbitrary expressions*.\n\nThe `index_function` can be an anonymous function or a binary representation obtained from the `function` field of [index_status](http://rethinkdb.com/api/python/index_status).\n\nIf successful, `create_index` will return an object of the form `{"created": 1}`. If an index by that name already exists on the table, a `RqlRuntimeError` will be thrown.\n\n*Example* Create a simple index based on the field `post_id`.\n\n r.table(\'comments\').index_create(\'post_id\').run(conn)\n*Example* Create a simple index based on the nested field `author > name`.\n\n r.table(\'comments\').index_create(\'author_name\', r.row["author"]["name"]).run(conn)\n\n*Example* Create a geospatial index based on the field `location`.\n\n r.table(\'places\').index_create(\'location\', geo=True).run(conn)\n\nA geospatial index field should contain only geometry objects. It will work with geometry ReQL terms ([get_intersecting](http://rethinkdb.com/api/python/get_intersecting/) and [get_nearest](http://rethinkdb.com/api/python/get_nearest/)) as well as index-specific terms ([index_status](http://rethinkdb.com/api/python/index_status), [index_wait](http://rethinkdb.com/api/python/index_wait), [index_drop](http://rethinkdb.com/api/python/index_drop) and [index_list](http://rethinkdb.com/api/python/index_list)). Using terms that rely on non-geometric ordering such as [get_all](http://rethinkdb.com/api/python/get_all/), [order_by](http://rethinkdb.com/api/python/order_by/) and [between](http://rethinkdb.com/api/python/order_by/) will result in an error.\n\n*Example* Create a compound index based on the fields `post_id` and `date`.\n\n r.table(\'comments\').index_create(\'post_and_date\', [r.row["post_id"], r.row["date"]]).run(conn)\n\n*Example* Create a multi index based on the field `authors`.\n\n r.table(\'posts\').index_create(\'authors\', multi=True).run(conn)\n\n*Example* Create a geospatial multi index based on the field `towers`.\n\n r.table(\'networks\').index_create(\'towers\', geo=True, multi=True).run(conn)\n\n*Example* Create an index based on an arbitrary expression.\n\n r.table(\'posts\').index_create(\'authors\', lambda doc:\n r.branch(\n doc.has_fields("updated_at"),\n doc["updated_at"],\n doc["created_at"]\n )\n ).run(conn)\n\n*Example* Create a new secondary index based on an existing one.\n\n index = r.table(\'posts\').index_status(\'authors\').nth(0)[\'function\'].run(conn)\n r.table(\'new_posts\').index_create(\'authors\', index).run(conn)\n\n*Example* Rebuild an outdated secondary index on a table.\n\n old_index = r.table(\'posts\').index_status(\'old_index\').nth(0)[\'function\'].run(conn)\n r.table(\'posts\').index_create(\'new_index\', old_index).run(conn)\n r.table(\'posts\').index_wait(\'new_index\').run(conn)\n r.table(\'posts\').index_rename(\'new_index\', \'old_index\', overwrite=True).run(conn)\n'), + (rethinkdb.ast.Table.index_drop, b"table.index_drop(index_name) -> object\n\nDelete a previously created secondary index of this table.\n\n*Example* Drop a secondary index named 'code_name'.\n\n r.table('dc').index_drop('code_name').run(conn)\n\n"), + (rethinkdb.ast.Table.index_list, b"table.index_list() -> array\n\nList all the secondary indexes of this table.\n\n*Example* List the available secondary indexes for this table.\n\n r.table('marvel').index_list().run(conn)\n"), + (rethinkdb.ast.Table.index_rename, b"table.index_rename(old_index_name, new_index_name[, overwrite=False]) -> object\n\nRename an existing secondary index on a table. If the optional argument `overwrite` is specified as `True`, a previously existing index with the new name will be deleted and the index will be renamed. If `overwrite` is `False` (the default) an error will be raised if the new index name already exists.\n\nThe return value on success will be an object of the format `{'renamed': 1}`, or `{'renamed': 0}` if the old and new names are the same.\n\nAn error will be raised if the old index name does not exist, if the new index name is already in use and `overwrite` is `False`, or if either the old or new index name are the same as the primary key field name.\n\n*Example* Rename an index on the comments table.\n\n r.table('comments').index_rename('post_id', 'message_id').run(conn)\n"), + (rethinkdb.ast.Table.index_status, b'table.index_status([, index...]) -> array\n\nGet the status of the specified indexes on this table, or the status\nof all indexes on this table if no indexes are specified.\n\nThe result is an array where for each index, there will be an object like this one:\n\n {\n "index": ,\n "ready": True,\n "function": ,\n "multi": ,\n "outdated": \n }\n\nor this one:\n\n {\n "index": ,\n "ready": False,\n "blocks_processed": ,\n "blocks_total": ,\n "function": ,\n "multi": ,\n "outdated": \n }\n\nThe `multi` field will be `true` or `false` depending on whether this index was created as a multi index (see [index_create](http://rethinkdb.com/api/python/index_create/) for details). The `outdated` field will be true if the index is outdated in the current version of RethinkDB and needs to be rebuilt.\n\nThe `function` field is a binary object containing an opaque representation of the secondary index (including the `multi` argument if specified). It can be passed as the second argument to [index_create](http://rethinkdb.com/api/python/index_create/) to create a new index with the same function; see `index_create` for more information.\n\n*Example* Get the status of all the indexes on `test`:\n\n r.table(\'test\').index_status().run(conn)\n\n*Example* Get the status of the `timestamp` index:\n\n r.table(\'test\').index_status(\'timestamp\').run(conn)\n\n*Example* Save the binary representation of the index:\n\n func = r.table(\'test\').index_status(\'timestamp\').nth(0)[\'function\'].run(conn)\n'), + (rethinkdb.ast.Table.index_wait, b'table.index_wait([, index...]) -> array\n\nWait for the specified indexes on this table to be ready, or for all\nindexes on this table to be ready if no indexes are specified.\n\nThe result is an array containing one object for each table index:\n\n {\n "index": ,\n "ready": True,\n "function": ,\n "multi": ,\n "geo": ,\n "outdated": \n }\n\nSee the [index_status](http://rethinkdb.com/api/python/index_status) documentation for a description of the field values.\n\n*Example* Wait for all indexes on the table `test` to be ready:\n\n r.table(\'test\').index_wait().run(conn)\n\n*Example* Wait for the index `timestamp` to be ready:\n\n r.table(\'test\').index_wait(\'timestamp\').run(conn)\n'), + (rethinkdb.ast.DB.table_create, b'db.table_create(table_name[, options]) -> object\n\nCreate a table. A RethinkDB table is a collection of JSON documents.\n\nIf successful, the command returns an object with two fields:\n\n* `tables_created`: always `1`.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: always `None`.\n * `new_val`: the table\'s new [config](http://rethinkdb.com/api/python/config) value.\n\nIf a table with the same name already exists, the command throws `RqlRuntimeError`.\n\nNote: Only alphanumeric characters and underscores are valid for the table name.\n\nWhen creating a table you can specify the following options:\n\n* `primary_key`: the name of the primary key. The default primary key is `id`.\n* `durability`: if set to `soft`, writes will be acknowledged by the server immediately and flushed to disk in the background. The default is `hard`: acknowledgment of writes happens after data has been written to disk.\n* `shards`: the number of shards, an integer from 1-32. Defaults to `1`.\n* `replicas`: either an integer or a mapping object. Defaults to `1`.\n * If `replicas` is an integer, it specifies the number of replicas per shard. Specifying more replicas than there are servers will return an error.\n * If `replicas` is an object, it specifies key-value pairs of server tags and the number of replicas to assign to those servers: `{\'tag1\': 2, \'tag2\': 4, \'tag3\': 2, ...}`.\n* `primary_replica_tag`: the primary server specified by its server tag. Required if `replicas` is an object; the tag must be in the object. This must *not* be specified if `replicas` is an integer.\n\nThe [data type](http://rethinkdb.com/docs/data-types/) of a primary key is usually a string (like a UUID) or a number, but it can also be a time, binary object, boolean or an array. It cannot be an object.\n\n*Example* Create a table named \'dc_universe\' with the default settings.\n\n r.db(\'test\').table_create(\'dc_universe\').run(conn)\n \n {\n "config_changes": [\n {\n "new_val": {\n "db": "test",\n "durability": "hard",\n "id": "20ea60d4-3b76-4817-8828-98a236df0297",\n "name": "dc_universe",\n "primary_key": "id",\n "shards": [\n {\n "primary_replica": "rethinkdb_srv1",\n "replicas": [\n "rethinkdb_srv1",\n "rethinkdb_srv2"\n ]\n }\n ],\n "write_acks": "majority"\n },\n "old_val": None\n }\n ],\n "tables_created": 1\n }\n\n*Example* Create a table named \'dc_universe\' using the field \'name\' as primary key.\n\n r.db(\'test\').table_create(\'dc_universe\', primary_key=\'name\').run(conn)\n\n*Example* Create a table set up for two shards and three replicas per shard. This requires three available servers.\n\n r.db(\'test\').table_create(\'dc_universe\', shards=2, replicas=3).run(conn)\n\nRead [Sharding and replication](http://rethinkdb.com/docs/sharding-and-replication/) for a complete discussion of the subject, including advanced topics.\n'), + (rethinkdb.ast.DB.table_drop, b'db.table_drop(table_name) -> object\n\nDrop a table. The table and all its data will be deleted.\n\nIf successful, the command returns an object with two fields:\n\n* `tables_dropped`: always `1`.\n* `config_changes`: a list containing one two-field object, `old_val` and `new_val`:\n * `old_val`: the dropped table\'s [config](http://rethinkdb.com/api/python/config) value.\n * `new_val`: always `None`.\n\nIf the given table does not exist in the database, the command throws `RqlRuntimeError`.\n\n*Example* Drop a table named \'dc_universe\'.\n\n r.db(\'test\').table_drop(\'dc_universe\').run(conn)\n \n {\n "config_changes": [\n {\n "old_val": {\n "db": "test",\n "durability": "hard",\n "id": "20ea60d4-3b76-4817-8828-98a236df0297",\n "name": "dc_universe",\n "primary_key": "id",\n "shards": [\n {\n "primary_replica": "rethinkdb_srv1",\n "replicas": [\n "rethinkdb_srv1",\n "rethinkdb_srv2"\n ]\n }\n ],\n "write_acks": "majority"\n },\n "new_val": None\n }\n ],\n "tables_dropped": 1\n }\n'), + (rethinkdb.ast.DB.table_list, b"db.table_list() -> array\n\nList all table names in a database. The result is a list of strings.\n\n*Example* List all tables of the 'test' database.\n\n r.db('test').table_list().run(conn)\n \n"), + (rethinkdb.ast.RqlQuery.__add__, b'number + number -> number\nstring + string -> string\narray + array -> array\ntime + number -> time\n\nSum two numbers, concatenate two strings, or concatenate 2 arrays.\n\n*Example* It\'s as easy as 2 + 2 = 4.\n\n > (r.expr(2) + 2).run(conn)\n \n 4\n\n*Example* Strings can be concatenated too.\n\n > (r.expr("foo") + "bar").run(conn)\n \n "foobar"\n\n*Example* Arrays can be concatenated too.\n\n > (r.expr(["foo", "bar"]) + ["buzz"]).run(conn)\n \n [\'foo\', \'bar\', \'buzz\']\n\n*Example* Create a date one year from now.\n\n r.now() + 365*24*60*60\n\n*Example* Use [args](http://rethinkdb.com/api/python/args) with `add` to sum multiple values.\n\n > r.add(r.args([10, 20, 30])).run(conn)\n \n 60\n\n*Example* Concatenate an array of strings with `args`.\n\n > r.add(r.args([\'foo\', \'bar\', \'buzz\'])).run(conn)\n \n "foobarbuzz"\n'), + (rethinkdb.add, b'number + number -> number\nstring + string -> string\narray + array -> array\ntime + number -> time\n\nSum two numbers, concatenate two strings, or concatenate 2 arrays.\n\n*Example* It\'s as easy as 2 + 2 = 4.\n\n > (r.expr(2) + 2).run(conn)\n \n 4\n\n*Example* Strings can be concatenated too.\n\n > (r.expr("foo") + "bar").run(conn)\n \n "foobar"\n\n*Example* Arrays can be concatenated too.\n\n > (r.expr(["foo", "bar"]) + ["buzz"]).run(conn)\n \n [\'foo\', \'bar\', \'buzz\']\n\n*Example* Create a date one year from now.\n\n r.now() + 365*24*60*60\n\n*Example* Use [args](http://rethinkdb.com/api/python/args) with `add` to sum multiple values.\n\n > r.add(r.args([10, 20, 30])).run(conn)\n \n 60\n\n*Example* Concatenate an array of strings with `args`.\n\n > r.add(r.args([\'foo\', \'bar\', \'buzz\'])).run(conn)\n \n "foobarbuzz"\n'), + (rethinkdb.ast.RqlQuery.__and__, b'bool & bool -> bool\nr.and_(bool, bool) -> bool\nbool.and_(bool) -> bool\n\nCompute the logical "and" of two or more values. The `and_` command can be used as an infix operator after its first argument (`r.expr(True).and_(False)`) or given all of its arguments as parameters (`r.and_(True, False)`). The standard Python and operator, `&`, may also be used with ReQL.\n\n*Example* Return whether both `a` and `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) & b).run(conn)\n \n False\n*Example* Return whether all of `x`, `y` and `z` evaluate to true.\n\n > x = True\n > y = True\n > z = True\n > r.and_(x, y, z).run(conn)\n \n True\n'), + (rethinkdb.and_, b'bool & bool -> bool\nr.and_(bool, bool) -> bool\nbool.and_(bool) -> bool\n\nCompute the logical "and" of two or more values. The `and_` command can be used as an infix operator after its first argument (`r.expr(True).and_(False)`) or given all of its arguments as parameters (`r.and_(True, False)`). The standard Python and operator, `&`, may also be used with ReQL.\n\n*Example* Return whether both `a` and `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) & b).run(conn)\n \n False\n*Example* Return whether all of `x`, `y` and `z` evaluate to true.\n\n > x = True\n > y = True\n > z = True\n > r.and_(x, y, z).run(conn)\n \n True\n'), + (rethinkdb.ast.RqlQuery.__div__, b"number / number -> number\n\nDivide two numbers.\n\n*Example* It's as easy as 2 / 2 = 1.\n\n (r.expr(2) / 2).run(conn)\n"), + (rethinkdb.div, b"number / number -> number\n\nDivide two numbers.\n\n*Example* It's as easy as 2 / 2 = 1.\n\n (r.expr(2) / 2).run(conn)\n"), + (rethinkdb.ast.RqlQuery.__eq__, b'value == value -> bool\nvalue.eq(value) -> bool\n\nTest if two values are equal.\n\n*Example* Does 2 equal 2?\n\n (r.expr(2) == 2).run(conn)\n r.expr(2).eq(2).run(conn)\n'), + (rethinkdb.ast.RqlQuery.eq, b'value == value -> bool\nvalue.eq(value) -> bool\n\nTest if two values are equal.\n\n*Example* Does 2 equal 2?\n\n (r.expr(2) == 2).run(conn)\n r.expr(2).eq(2).run(conn)\n'), + (rethinkdb.ast.RqlQuery.__ge__, b'value >= value -> bool\nvalue.ge(value) -> bool\n\nTest if the first value is greater than or equal to other.\n\n*Example* Is 2 greater than or equal to 2?\n\n (r.expr(2) >= 2).run(conn)\n r.expr(2).ge(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.ge, b'value >= value -> bool\nvalue.ge(value) -> bool\n\nTest if the first value is greater than or equal to other.\n\n*Example* Is 2 greater than or equal to 2?\n\n (r.expr(2) >= 2).run(conn)\n r.expr(2).ge(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__gt__, b'value > value -> bool\nvalue.gt(value) -> bool\n\nTest if the first value is greater than other.\n\n*Example* Is 2 greater than 2?\n\n (r.expr(2) > 2).run(conn)\n r.expr(2).gt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.gt, b'value > value -> bool\nvalue.gt(value) -> bool\n\nTest if the first value is greater than other.\n\n*Example* Is 2 greater than 2?\n\n (r.expr(2) > 2).run(conn)\n r.expr(2).gt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__le__, b'value <= value -> bool\nvalue.le(value) -> bool\n\nTest if the first value is less than or equal to other.\n\n*Example* Is 2 less than or equal to 2?\n\n (r.expr(2) <= 2).run(conn)\n r.expr(2).le(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.le, b'value <= value -> bool\nvalue.le(value) -> bool\n\nTest if the first value is less than or equal to other.\n\n*Example* Is 2 less than or equal to 2?\n\n (r.expr(2) <= 2).run(conn)\n r.expr(2).le(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__lt__, b'value < value -> bool\nvalue.lt(value) -> bool\n\nTest if the first value is less than other.\n\n*Example* Is 2 less than 2?\n\n (r.expr(2) < 2).run(conn)\n r.expr(2).lt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.lt, b'value < value -> bool\nvalue.lt(value) -> bool\n\nTest if the first value is less than other.\n\n*Example* Is 2 less than 2?\n\n (r.expr(2) < 2).run(conn)\n r.expr(2).lt(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__mod__, b"number % number -> number\n\nFind the remainder when dividing two numbers.\n\n*Example* It's as easy as 2 % 2 = 0.\n\n (r.expr(2) % 2).run(conn)\n\n`\n"), + (rethinkdb.mod, b"number % number -> number\n\nFind the remainder when dividing two numbers.\n\n*Example* It's as easy as 2 % 2 = 0.\n\n (r.expr(2) % 2).run(conn)\n\n`\n"), + (rethinkdb.ast.RqlQuery.__mul__, b'number * number -> number\narray * number -> array\n\nMultiply two numbers, or make a periodic array.\n\n*Example* It\'s as easy as 2 * 2 = 4.\n\n (r.expr(2) * 2).run(conn)\n\n*Example* Arrays can be multiplied by numbers as well.\n\n (r.expr(["This", "is", "the", "song", "that", "never", "ends."]) * 100).run(conn)\n\n'), + (rethinkdb.mul, b'number * number -> number\narray * number -> array\n\nMultiply two numbers, or make a periodic array.\n\n*Example* It\'s as easy as 2 * 2 = 4.\n\n (r.expr(2) * 2).run(conn)\n\n*Example* Arrays can be multiplied by numbers as well.\n\n (r.expr(["This", "is", "the", "song", "that", "never", "ends."]) * 100).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__ne__, b'value != value -> bool\nvalue.ne(value) -> bool\n\nTest if two values are not equal.\n\n*Example* Does 2 not equal 2?\n\n (r.expr(2) != 2).run(conn)\n r.expr(2).ne(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.ne, b'value != value -> bool\nvalue.ne(value) -> bool\n\nTest if two values are not equal.\n\n*Example* Does 2 not equal 2?\n\n (r.expr(2) != 2).run(conn)\n r.expr(2).ne(2).run(conn)\n\n'), + (rethinkdb.ast.RqlQuery.__invert__, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.not_, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), + (rethinkdb.not_, b'bool.not_() -> bool\nnot_(bool) -> bool\n(~bool) -> bool\n\nCompute the logical inverse (not) of an expression.\n\n`not_` can be called either via method chaining, immediately after an expression that evaluates as a boolean value, or by passing the expression as a parameter to `not_`. All values that are not `False` or `None` will be converted to `True`.\n\nYou may also use `~` as a shorthand operator.\n\n*Example* Not true is false.\n\n r.not_(True).run(conn)\n r.expr(True).not_().run(conn)\n (~r.expr(True)).run(conn)\n\nThese evaluate to `false`.\n\nNote that when using `~` the expression is wrapped in parentheses. Without this, Python will evaluate `r.expr(True)` *first* rather than using the ReQL operator and return an incorrect value. (`~True` evaluates to −2 in Python.)\n\n*Example* Return all the users that do not have a "flag" field.\n\n r.table(\'users\').filter(\n lambda users: (~users.has_fields(\'flag\'))\n ).run(conn)\n\n*Example* As above, but prefix-style.\n\n r.table(\'users\').filter(\n lambda users: r.not_(users.has_fields(\'flag\'))\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.__or__, b'bool | bool -> bool\nbool.or_(bool[, bool, ...]) -> bool\nr.or_(bool, bool) -> bool\n\nCompute the logical "or" of two or more values. The `or_` command can be used as an infix operator after its first argument (`r.expr(True).or_(False)`) or given all of its arguments as parameters (`r.or_(True, False)`). The standard Python or operator, `|`, may also be used with ReQL.\n\n*Example* Return whether either `a` or `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) | b).run(conn)\n \n True\n\n*Example* Return whether any of `x`, `y` or `z` evaluate to true.\n\n > x = False\n > y = False\n > z = False\n > r.or_(x, y, z).run(conn)\n \n False\n\n__Note:__ When using `or` inside a `filter` predicate to test the values of fields that may not exist on the documents being tested, you should use the `default` command with those fields so they explicitly return `False`.\n\n r.table(\'posts\').filter(lambda post:\n post[\'category\'].default(\'foo\').eq(\'article\').or(\n post[\'genre\'].default(\'foo\').eq(\'mystery\'))\n ).run(conn)\n'), + (rethinkdb.or_, b'bool | bool -> bool\nbool.or_(bool[, bool, ...]) -> bool\nr.or_(bool, bool) -> bool\n\nCompute the logical "or" of two or more values. The `or_` command can be used as an infix operator after its first argument (`r.expr(True).or_(False)`) or given all of its arguments as parameters (`r.or_(True, False)`). The standard Python or operator, `|`, may also be used with ReQL.\n\n*Example* Return whether either `a` or `b` evaluate to true.\n\n > a = True\n > b = False\n > (r.expr(a) | b).run(conn)\n \n True\n\n*Example* Return whether any of `x`, `y` or `z` evaluate to true.\n\n > x = False\n > y = False\n > z = False\n > r.or_(x, y, z).run(conn)\n \n False\n\n__Note:__ When using `or` inside a `filter` predicate to test the values of fields that may not exist on the documents being tested, you should use the `default` command with those fields so they explicitly return `False`.\n\n r.table(\'posts\').filter(lambda post:\n post[\'category\'].default(\'foo\').eq(\'article\').or(\n post[\'genre\'].default(\'foo\').eq(\'mystery\'))\n ).run(conn)\n'), + (rethinkdb.random, b"r.random() -> number\nr.random(number[, number], float=True) -> number\nr.random(integer[, integer]) -> integer\n\nGenerate a random number between given (or implied) bounds. `random` takes zero, one or two arguments.\n\n- With __zero__ arguments, the result will be a floating-point number in the range `[0,1)` (from 0 up to but not including 1).\n- With __one__ argument _x,_ the result will be in the range `[0,x)`, and will be integer unless `float=True` is given as an option. Specifying a floating point number without the `float` option will raise an error.\n- With __two__ arguments _x_ and _y,_ the result will be in the range `[x,y)`, and will be integer unless `float=True` is given as an option. If _x_ and _y_ are equal an error will occur, unless the floating-point option has been specified, in which case _x_ will be returned. Specifying a floating point number without the `float` option will raise an error.\n\nNote: The last argument given will always be the 'open' side of the range, but when\ngenerating a floating-point number, the 'open' side may be less than the 'closed' side.\n\n*Example* Generate a random number in the range `[0,1)`\n\n r.random().run(conn)\n\n*Example* Generate a random integer in the range `[0,100)`\n\n r.random(100).run(conn)\n r.random(0, 100).run(conn)\n\n*Example* Generate a random number in the range `(-2.24,1.59]`\n\n r.random(1.59, -2.24, float=True).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.__sub__, b"number - number -> number\ntime - time -> number\ntime - number -> time\n\nSubtract two numbers.\n\n*Example* It's as easy as 2 - 2 = 0.\n\n (r.expr(2) - 2).run(conn)\n\n*Example* Create a date one year ago today.\n\n r.now() - 365*24*60*60\n\n*Example* Retrieve how many seconds elapsed between today and date\n\n r.now() - date\n\n"), + (rethinkdb.sub, b"number - number -> number\ntime - time -> number\ntime - number -> time\n\nSubtract two numbers.\n\n*Example* It's as easy as 2 - 2 = 0.\n\n (r.expr(2) - 2).run(conn)\n\n*Example* Create a date one year ago today.\n\n r.now() - 365*24*60*60\n\n*Example* Retrieve how many seconds elapsed between today and date\n\n r.now() - date\n\n"), + (rethinkdb.ast.Table.between, b'table.between(lower_key, upper_key[, index=\'id\', left_bound=\'closed\', right_bound=\'open\'])\n -> selection\n\nGet all documents between two keys. Accepts three optional arguments: `index`, `left_bound`, and `right_bound`. If `index` is set to the name of a secondary index, `between` will return all documents where that index\'s value is in the specified range (it uses the primary key by default). `left_bound` or `right_bound` may be set to `open` or `closed` to indicate whether or not to include that endpoint of the range (by default, `left_bound` is closed and `right_bound` is open).\n\nYou may also use the special constants `r.minval` and `r.maxval` for boundaries, which represent "less than any index key" and "more than any index key" respectively. For instance, if you use `r.minval` as the lower key, then `between` will return all documents whose primary keys (or indexes) are less than the specified upper key.\n\nNote that compound indexes are sorted using [lexicographical order][lo]. Take the following range as an example:\n\n\t[[1, "c"] ... [5, "e"]]\n\nThis range includes all compound keys:\n\n* whose first item is 1 and second item is equal or greater than "c";\n* whose first item is between 1 and 5, *regardless of the value of the second item*;\n* whose first item is 5 and second item is less than or equal to "e".\n\n[lo]: https://en.wikipedia.org/wiki/Lexicographical_order\n\n*Example* Find all users with primary key >= 10 and < 20 (a normal half-open interval).\n\n r.table(\'marvel\').between(10, 20).run(conn)\n\n*Example* Find all users with primary key >= 10 and <= 20 (an interval closed on both sides).\n\n r.table(\'marvel\').between(10, 20, right_bound=\'closed\').run(conn)\n\n*Example* Find all users with primary key < 20.\n\n r.table(\'marvel\').between(r.minval, 20).run(conn)\n\n*Example* Find all users with primary key > 10.\n\n r.table(\'marvel\').between(10, r.maxval, left_bound=\'open\').run(conn)\n\n*Example* Between can be used on secondary indexes too. Just pass an optional index argument giving the secondary index to query.\n\n r.table(\'dc\').between(\'dark_knight\', \'man_of_steel\', index=\'code_name\').run(conn)\n\n*Example* Get all users whose full name is between "John Smith" and "Wade Welles."\n\n r.table("users").between(["Smith", "John"], ["Welles", "Wade"],\n index="full_name").run(conn)\n\n*Example* Subscribe to a [changefeed](http://rethinkdb.com/docs/changefeeds/javascript) of teams ranked in the top 10.\n\n changes = r.table("teams").between(1, 11, index="rank").changes().run(conn)\n\n__Note:__ Between works with secondary indexes on date fields, but will not work with unindexed date fields. To test whether a date value is between two other dates, use the [during](http://rethinkdb.com/api/python/during) command, not `between`.\n\nSecondary indexes can be used in extremely powerful ways with `between` and other commands; read the full article on [secondary indexes](http://rethinkdb.com/docs/secondary-indexes) for examples using boolean operations, `contains` and more.\n\n__Note:__ RethinkDB uses byte-wise ordering for `between` and does not support Unicode collations; non-ASCII characters will be sorted by UTF-8 codepoint.\n\n__Note:__ If you chain `between` after [order_by](http://rethinkdb.com/api/python/order_by), the `between` command must use the index specified in `order_by`, and will default to that index. Trying to specify another index will result in a `RqlRuntimeError`.\n'), + (rethinkdb.db, b"r.db(db_name) -> db\n\nReference a database.\n\n*Example* Before we can query a table we have to select the correct database.\n\n r.db('heroes').table('marvel').run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.filter, b'selection.filter(predicate[, default=False]) -> selection\nstream.filter(predicate[, default=False]) -> stream\narray.filter(predicate[, default=False]) -> array\n\nReturn all the elements in a sequence for which the given predicate is true. The return value of `filter` will be the same as the input (sequence, stream, or array). Documents can be filtered in a variety of ways—ranges, nested values, boolean conditions, and the results of anonymous functions.\n\nBy default, `filter` will silently skip documents with missing fields: if the predicate tries to access a field that doesn\'t exist (for instance, the predicate `{\'age\': 30}` applied to a document with no `age` field), that document will not be returned in the result set, and no error will be generated. This behavior can be changed with the `default` optional argument.\n\n* If `default` is set to `True`, documents with missing fields will be returned rather than skipped.\n* If `default` is set to `r.error()`, an `RqlRuntimeError` will be thrown when a document with a missing field is tested.\n* If `default` is set to `False` (the default), documents with missing fields will be skipped.\n\n*Example* Get all users who are 30 years old.\n\n r.table(\'users\').filter({\'age\': 30}).run(conn)\n\nThe predicate `{\'age\': 30}` selects documents in the `users` table with an `age` field whose value is `30`. Documents with an `age` field set to any other value *or* with no `age` field present are skipped.\n\nWhile the `{\'field\': value}` style of predicate is useful for exact matches, a more general way to write a predicate is to use the [row](http://rethinkdb.com/api/python/row) command with a comparison operator such as [eq](http://rethinkdb.com/api/python/eq) (`==`) or [gt](http://rethinkdb.com/api/python/gt) (`>`), or to use a lambda function that returns `True` or `False`.\n\n r.table(\'users\').filter(r.row["age"] == 30).run(conn)\n\nIn this case, the predicate `r.row["age"] == 30` returns `True` if the field `age` is equal to 30. You can write this predicate as a lambda function instead:\n\n r.table(\'users\').filter(lambda user:\n user["age"] == 30\n ).run(conn)\n\nPredicates to `filter` are evaluated on the server, and must use ReQL expressions. Some Python comparison operators are overloaded by the RethinkDB driver and will be translated to ReQL, such as `==`, `<`/`>` and `|`/`&` (note the single character form, rather than `||`/`&&`).\n\nAlso, predicates must evaluate document fields. They cannot evaluate [secondary indexes](http://rethinkdb.com/docs/secondary-indexes/).\n\n*Example* Get all users who are more than 18 years old.\n\n r.table("users").filter(r.row["age"] > 18).run(conn)\n\n*Example* Get all users who are less than 18 years old and more than 13 years old.\n\n r.table("users").filter((r.row["age"] < 18) & (r.row["age"] > 13)).run(conn)\n\n*Example* Get all users who are more than 18 years old or have their parental consent.\n\n r.table("users").filter(\n (r.row["age"] >= 18) | (r.row["hasParentalConsent"])).run(conn)\n\n*Example* Retrieve all users who subscribed between January 1st, 2012\n(included) and January 1st, 2013 (excluded).\n\n r.table("users").filter(\n lambda user: user["subscription_date"].during(\n r.time(2012, 1, 1, \'Z\'), r.time(2013, 1, 1, \'Z\'))\n ).run(conn)\n\n*Example* Retrieve all users who have a gmail account (whose field `email` ends with `@gmail.com`).\n\n r.table("users").filter(\n lambda user: user["email"].match("@gmail.com$")\n ).run(conn)\n\n*Example* Filter based on the presence of a value in an array.\n\nGiven this schema for the `users` table:\n\n {\n "name": \n "places_visited": []\n }\n\nRetrieve all users whose field `places_visited` contains `France`.\n\n r.table("users").filter(lambda user:\n user["places_visited"].contains("France")\n ).run(conn)\n\n*Example* Filter based on nested fields.\n\nGiven this schema for the `users` table:\n\n {\n "id": \n "name": {\n "first": ,\n "middle": ,\n "last": \n }\n }\n\nRetrieve all users named "William Adama" (first name "William", last name\n"Adama"), with any middle name.\n\n r.table("users").filter({\n "name": {\n "first": "William",\n "last": "Adama"\n }\n }).run(conn)\n\nIf you want an exact match for a field that is an object, you will have to use `r.literal`.\n\nRetrieve all users named "William Adama" (first name "William", last name\n"Adama"), and who do not have a middle name.\n\n r.table("users").filter(r.literal({\n "name": {\n "first": "William",\n "last": "Adama"\n }\n })).run(conn)\n\nYou may rewrite these with lambda functions.\n\n r.table("users").filter(\n lambda user:\n (user["name"]["first"] == "William")\n & (user["name"]["last"] == "Adama")\n ).run(conn)\n\n r.table("users").filter(lambda user:\n user["name"] == {\n "first": "William",\n "last": "Adama"\n }\n ).run(conn)\n\nBy default, documents missing fields tested by the `filter` predicate are skipped. In the previous examples, users without an `age` field are not returned. By passing the optional `default` argument to `filter`, you can change this behavior.\n\n*Example* Get all users less than 18 years old or whose `age` field is missing.\n\n r.table("users").filter(r.row["age"] < 18, default=True).run(conn)\n\n*Example* Get all users more than 18 years old. Throw an error if a\ndocument is missing the field `age`.\n\n r.table("users").filter(r.row["age"] > 18, default=r.error()).run(conn)\n\n*Example* Get all users who have given their phone number (all the documents whose field `phone_number` exists and is not `None`).\n\n r.table(\'users\').filter(\n lambda user: user.has_fields(\'phone_number\')\n ).run(conn)\n\n*Example* Get all users with an "editor" role or an "admin" privilege.\n\n r.table(\'users\').filter(\n lambda user: (user[\'role\'] == \'editor\').default(False) |\n (user[\'privilege\'] == \'admin\').default(False)\n ).run(conn)\n\nInstead of using the `default` optional argument to `filter`, we have to use default values on the fields within the `or` clause. Why? If the field on the left side of the `or` clause is missing from a document—in this case, if the user doesn\'t have a `role` field—the predicate will generate an error, and will return `False` (or the value the `default` argument is set to) without evaluating the right side of the `or`. By using `.default(False)` on the fields, each side of the `or` will evaluate to either the field\'s value or `False` if the field doesn\'t exist.\n'), + (rethinkdb.ast.Table.get, b"table.get(key) -> singleRowSelection\n\nGet a document by primary key.\n\nIf no document exists with that primary key, `get` will return `None`.\n\n*Example* Find a document by UUID.\n\n r.table('posts').get('a9849eef-7176-4411-935b-79a6e3c56a74').run(conn)\n\n*Example* Find a document and merge another document with it.\n\n r.table('heroes').get(3).merge(\n { 'powers': ['invisibility', 'speed'] }\n ).run(conn)\n\n_*Example* Subscribe to a document's [changefeed](http://rethinkdb.com/docs/changefeeds/python).\n\n changes = r.table('heroes').get(3).changes().run(conn)\n"), + (rethinkdb.ast.Table.get_all, b"table.get_all(key1[, key2...], [, index='id']) -> selection\n\nGet all documents where the given value matches the value of the requested index.\n\n*Example* Secondary index keys are not guaranteed to be unique so we cannot query via [get](http://rethinkdb.com/api/python/get/) when using a secondary index.\n\n r.table('marvel').get_all('man_of_steel', index='code_name').run(conn)\n\n*Example* Without an index argument, we default to the primary index. While `get` will either return the document or `None` when no document with such a primary key value exists, this will return either a one or zero length stream.\n\n r.table('dc').get_all('superman').run(conn)\n\n*Example* You can get multiple documents in a single call to `get_all`.\n\n r.table('dc').get_all('superman', 'ant man').run(conn)\n\n*Example* You can use [args](http://rethinkdb.com/api/python/args/) with `get_all` to retrieve multiple documents whose keys are in a list. This uses `get_all` to get a list of female superheroes, coerces that to an array, and then gets a list of villains who have those superheroes as enemies.\n\n r.do(\n r.table('heroes').get_all('f', {'index': 'gender'})['id'].coerce_to('array'), \n lamdba heroines: r.table('villains').get_all(r.args(heroines))\n ).run(conn)\n\nSecondary indexes can be used in extremely powerful ways with `get_all` and other commands; read the full article on [secondary indexes](http://rethinkdb.com/docs/secondary-indexes) for examples using boolean operations, `contains` and more.\n"), + (rethinkdb.ast.DB.table, b"db.table(name[, use_outdated=False, identifier_format='name']) -> table\n\nReturn all documents in a table. Other commands may be chained after `table` to return a subset of documents (such as `get` and `filter`) or perform further processing.\n\n*Example* Return all documents in the table 'marvel' of the default database.\n\n r.table('marvel').run(conn)\n\n*Example* Return all documents in the table 'marvel' of the database 'heroes'.\n\n r.db('heroes').table('marvel').run(conn)\n\nThere are two optional arguments.\n\n* `use_outdated`: if `True`, this allows potentially out-of-date data to be returned, with potentially faster reads. It also allows you to perform reads from a secondary replica if a primary has failed. Default `False`.\n* `identifier_format`: possible values are `name` and `uuid`, with a default of `name`. If set to `uuid`, then [system tables](http://rethinkdb.com/docs/system-tables/) will refer to servers, databases and tables by UUID rather than name. (This only has an effect when used with system tables.)\n\n*Example* Allow potentially out-of-date data in exchange for faster reads.\n\n r.db('heroes').table('marvel', use_outdated=True).run(conn)\n"), + (rethinkdb.ast.RqlQuery.downcase, b'string.downcase() -> string\n\nLowercases a string.\n\n*Example*\n\n > r.expr("Sentence about LaTeX.").downcase().run(conn)\n "sentence about latex."\n\n__Note:__ `upcase` and `downcase` only affect ASCII characters.\n'), + (rethinkdb.ast.RqlQuery.match, b'string.match(regexp) -> None/object\n\nMatches against a regular expression. If there is a match, returns an object with the fields:\n\n- `str`: The matched string\n- `start`: The matched string\'s start\n- `end`: The matched string\'s end\n- `groups`: The capture groups defined with parentheses\n\nIf no match is found, returns `None`.\n\nAccepts RE2 syntax\n([https://code.google.com/p/re2/wiki/Syntax](https://code.google.com/p/re2/wiki/Syntax)).\nYou can enable case-insensitive matching by prefixing the regular expression with\n`(?i)`. See the linked RE2 documentation for more flags.\n\nThe `match` command does not support backreferences.\n\n*Example* Get all users whose name starts with "A". Because `None` evaluates to `false` in\n`filter`, you can just use the result of `match` for the predicate.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("^A")\n ).run(conn)\n\n*Example* Get all users whose name ends with "n".\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("n$")\n ).run(conn)\n\n*Example* Get all users whose name has "li" in it\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("li")\n ).run(conn)\n\n*Example* Get all users whose name is "John" with a case-insensitive search.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("(?i)^john$")\n ).run(conn)\n\n*Example* Get all users whose name is composed of only characters between "a" and "z".\n\n r.table(\'users\').filter(lambda doc:\n doc[\'name\'].match("(?i)^[a-z]+$")\n ).run(conn)\n\n*Example* Get all users where the zipcode is a string of 5 digits.\n\n r.table(\'users\').filter(lambda doc:\n doc[\'zipcode\'].match("\\d{5}")\n ).run(conn)\n\n*Example* Retrieve the domain of a basic email\n\n r.expr("name@domain.com").match(".*@(.*)").run(conn)\n\nResult:\n\n {\n "start": 0,\n "end": 20,\n "str": "name@domain.com",\n "groups":[\n {\n "end": 17,\n "start": 7,\n "str": "domain.com"\n }\n ]\n }\n\nYou can then retrieve only the domain with the [\\[\\]](http://rethinkdb.com/api/python/get_field) selector.\n\n r.expr("name@domain.com").match(".*@(.*)")["groups"][0]["str"].run(conn)\n\nReturns `\'domain.com\'`\n\n*Example* Fail to parse out the domain and returns `None`.\n\n r.expr("name[at]domain.com").match(".*@(.*)").run(conn)\n'), + (rethinkdb.ast.RqlQuery.split, b'string.split([separator, [max_splits]]) -> array\n\nSplits a string into substrings. Splits on whitespace when called\nwith no arguments. When called with a separator, splits on that\nseparator. When called with a separator and a maximum number of\nsplits, splits on that separator at most `max_splits` times. (Can be\ncalled with `None` as the separator if you want to split on whitespace\nwhile still specifying `max_splits`.)\n\nMimics the behavior of Python\'s `string.split` in edge cases, except\nfor splitting on the empty string, which instead produces an array of\nsingle-character strings.\n\n*Example* Split on whitespace.\n\n > r.expr("foo bar bax").split().run(conn)\n ["foo", "bar", "bax"]\n\n*Example* Split the entries in a CSV file.\n\n > r.expr("12,37,,22,").split(",").run(conn)\n ["12", "37", "", "22", ""]\n\n*Example* Split a string into characters.\n\n > r.expr("mlucy").split("").run(conn)\n ["m", "l", "u", "c", "y"]\n\n*Example* Split the entries in a CSV file, but only at most 3\ntimes.\n\n > r.expr("12,37,,22,").split(",", 3).run(conn)\n ["12", "37", "", "22,"]\n\n*Example* Split on whitespace at most once (i.e. get the first word).\n\n > r.expr("foo bar bax").split(None, 1).run(conn)\n ["foo", "bar bax"]\n'), + (rethinkdb.ast.RqlQuery.upcase, b'string.upcase() -> string\n\nUppercases a string.\n\n*Example*\n\n > r.expr("Sentence about LaTeX.").upcase().run(conn)\n "SENTENCE ABOUT LATEX."\n\n__Note:__ `upcase` and `downcase` only affect ASCII characters.\n'), + (rethinkdb.ast.RqlQuery.concat_map, b'stream.concat_map(mapping_function) -> stream\narray.concat_map(mapping_function) -> array\n\nConcatenate one or more elements into a single sequence using a mapping function.\n\n`concat_map` works in a similar fashion to `map`, applying the given function to each element in a sequence, but it will always return a single sequence. If the mapping function returns a sequence, `map` would produce a sequence of sequences:\n\n r.expr([1, 2, 3]).map(lambda x: [x, x.mul(2)]).run(conn)\n\nResult:\n\n [[1, 2], [2, 4], [3, 6]]\n\nWhereas `concat_map` with the same mapping function would merge those sequences into one:\n\n r.expr([1, 2, 3]).concat_map(lambda x: [x, x.mul(2)]).run(conn)\n\nResult:\n\n [1, 2, 2, 4, 3, 6]\n\nThe return value, array or stream, will be the same type as the input.\n\n*Example* Construct a sequence of all monsters defeated by Marvel heroes. The field "defeatedMonsters" is an array of one or more monster names.\n\n r.table(\'marvel\').concat_map(lambda hero: hero[\'defeatedMonsters\']).run(conn)\n\n*Example* Simulate an [eq_join](http://rethinkdb.com/api/python/eq_join/) using `concat_map`. (This is how ReQL joins are implemented internally.)\n\n r.table(\'posts\').concat_map(\n lambda post: r.table(\'comments\').get_all(\n post[\'id\'], index=\'post_id\'\n ).map(\n lambda comment: { \'left\': post, \'right\': comment}\n )\n ).run(conn)\n'), + (rethinkdb.ast.RqlQuery.is_empty, b"sequence.is_empty() -> bool\n\nTest if a sequence is empty.\n\n*Example* Are there any documents in the marvel table?\n\n r.table('marvel').is_empty().run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.limit, b"sequence.limit(n) -> stream\narray.limit(n) -> array\n\nEnd the sequence after the given number of elements.\n\n*Example* Only so many can fit in our Pantheon of heroes.\n\n r.table('marvel').order_by('belovedness').limit(10).run(conn)\n"), + (rethinkdb.ast.RqlQuery.map, b"sequence1.map([sequence2, ...], mapping_function) -> stream\narray1.map([sequence2, ...], mapping_function) -> array\nr.map(sequence1[, sequence2, ...], mapping_function) -> stream\nr.map(array1[, array2, ...], mapping_function) -> array\n\nTransform each element of one or more sequences by applying a mapping function to them. If `map` is run with two or more sequences, it will iterate for as many items as there are in the shortest sequence.\n\nNote that `map` can only be applied to sequences, not single values. If you wish to apply a function to a single value/selection (including an array), use the [do](http://rethinkdb.com/api/python/do) command.\n\n*Example* Return the first five squares.\n\n > r.expr([1, 2, 3, 4, 5]).map(lambda val: (val * val)).run(conn)\n \n [1, 4, 9, 16, 25]\n\n*Example* Sum the elements of three sequences.\n\n > sequence1 = [100, 200, 300, 400]\n > sequence2 = [10, 20, 30, 40]\n > sequence3 = [1, 2, 3, 4]\n > r.map(sequence1, sequence2, sequence3,\n lambda val1, val2, val3: (val1 + val2 + val3)).run(conn)\n \n [111, 222, 333, 444]\n\n*Example* Rename a field when retrieving documents using `map` and `merge`.\n\nThis example renames the field `id` to `user_id` when retrieving documents from the table `users`.\n\n r.table('users').map(\n lambda doc: doc.merge({'user_id': doc['id']}).without('id')).run(conn)\n\nNote that in this case, [row](http://rethinkdb.com/api/python/row) may be used as an alternative to writing an anonymous function, as it returns the same value as the function parameter receives:\n\n r.table('users').map(\n r.row.merge({'user_id': r.row['id']}).without('id')).run(conn)\n\n*Example* Assign every superhero an archenemy.\n\n r.table('heroes').map(r.table('villains'),\n lambda hero, villain: hero.merge({'villain': villain})).run(conn)\n"), + (rethinkdb.ast.RqlQuery.nth, b"sequence.nth(index) -> object\nselection.nth(index) -> selection<object>\n\nGet the *nth* element of a sequence, counting from zero. If the argument is negative, count from the last element.\n\nIn Python, you can use `[]` with an integer as a shorthand for `nth`.\n\n*Example* Select the second element in the array.\n\n r.expr([1,2,3]).nth(1).run(conn)\n r.expr([1,2,3])[1].run(conn)\n\n*Example* Select the bronze medalist from the competitors.\n\n r.table('players').order_by(index=r.desc('score')).nth(3).run(conn)\n\n*Example* Select the last place competitor.\n\n r.table('players').order_by(index=r.desc('score')).nth(-1).run(conn)\n"), + (rethinkdb.ast.RqlQuery.offsets_of, b"sequence.offsets_of(datum | predicate) -> array\n\nGet the indexes of an element in a sequence. If the argument is a predicate, get the indexes of all elements matching it.\n\n*Example* Find the position of the letter 'c'.\n\n r.expr(['a','b','c']).offsets_of('c').run(conn)\n\n*Example* Find the popularity ranking of invisible heroes.\n\n r.table('marvel').union(r.table('dc')).order_by('popularity').offsets_of(\n r.row['superpowers'].contains('invisibility')\n ).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.order_by, b'table.order_by([key1...], index=index_name) -> selection\nselection.order_by(key1, [key2...]) -> selection\nsequence.order_by(key1, [key2...]) -> array\n\nSort the sequence by document values of the given key(s). To specify\nthe ordering, wrap the attribute with either `r.asc` or `r.desc`\n(defaults to ascending).\n\n__Note:__ RethinkDB uses byte-wise ordering for `orderBy` and does not support Unicode collations; non-ASCII characters will be sorted by UTF-8 codepoint. For more information on RethinkDB\'s sorting order, read the section in [ReQL data types](http://rethinkdb.com/docs/data-types/#sorting-order).\n\nSorting without an index requires the server to hold the sequence in\nmemory, and is limited to 100,000 documents (or the setting of the `arrayLimit` option for [run](http://rethinkdb.com/api/python/run)). Sorting with an index can\nbe done on arbitrarily large tables, or after a `between` command\nusing the same index.\n\n*Example* Order all the posts using the index `date`. \n\n r.table(\'posts\').order_by(index=\'date\').run(conn)\n\nThe index must either be the primary key or have been previously created with [index_create](http://rethinkdb.com/api/python/index_create/).\n\n r.table(\'posts\').index_create(\'date\').run(conn)\n\nYou can also select a descending ordering:\n\n r.table(\'posts\').order_by(index=r.desc(\'date\')).run(conn, callback)\n\n*Example* Order a sequence without an index.\n\n r.table(\'posts\').get(1)[\'comments\'].order_by(\'date\')\n\nYou can also select a descending ordering:\n\n r.table(\'posts\').get(1)[\'comments\'].order_by(r.desc(\'date\'))\n\nIf you\'re doing ad-hoc analysis and know your table won\'t have more then 100,000\nelements (or you\'ve changed the setting of the `arrayLimit` option for [run](http://rethinkdb.com/api/python/run)) you can run `order_by` without an index:\n\n r.table(\'small_table\').order_by(\'date\')\n\n*Example* You can efficiently order using multiple fields by using a\n[compound index](http://www.rethinkdb.com/docs/secondary-indexes/python/).\n\nOrder by date and title.\n\n r.table(\'posts\').order_by(index=\'date_and_title\').run(conn)\n\nThe index must have been previously created with [index_create](http://rethinkdb.com/api/python/index_create/).\n\n r.table(\'posts\').index_create(\'date_and_title\', lambda post:\n [post["date"], post["title"]]).run(conn)\n\n_Note_: You cannot specify multiple orders in a compound index. See [issue #2306](https://github.com/rethinkdb/rethinkdb/issues/2306)\nto track progress.\n\n*Example* If you have a sequence with fewer documents than the `array_limit`, you can order it\nby multiple fields without an index.\n\n r.table(\'small_table\').order_by(\'date\', r.desc(\'title\'))\n\n*Example* Notice that an index ordering always has highest\nprecedence. The following query orders posts by date, and if multiple\nposts were published on the same date, they will be ordered by title.\n\n r.table(\'post\').order_by(\'title\', index=\'date\').run(conn)\n*Example* You can use [nested field](http://rethinkdb.com/docs/cookbook/python/#filtering-based-on-nested-fields) syntax to sort on fields from subdocuments. (You can also create indexes on nested fields using this syntax with `index_create`.)\n\n r.table(\'user\').order_by(lambda user: user[\'group\'][\'id\']).run(conn)\n\n*Example* You can efficiently order data on arbitrary expressions using indexes.\n\n r.table(\'posts\').order_by(index=\'votes\').run(conn)\n\nThe index must have been previously created with [index_create](http://rethinkdb.com/api/ruby/index_create/).\n\n r.table(\'posts\').index_create(\'votes\', lambda post:\n post["upvotes"]-post["downvotes"]\n ).run(conn)\n\n*Example* If you have a sequence with fewer documents than the `array_limit`, you can order it with an arbitrary function directly.\n\n r.table(\'small_table\').order_by(lambda doc:\n doc[\'upvotes\']-doc[\'downvotes\']\n );\n\nYou can also select a descending ordering:\n\n r.table(\'small_table\').order_by(r.desc(lambda doc:\n doc[\'upvotes\']-doc[\'downvotes\']\n ));\n\n*Example* Ordering after a `between` command can be done as long as the same index is being used.\n\n r.table("posts").between(r.time(2013, 1, 1, \'+00:00\'), r.time(2013, 1, 1, \'+00:00\'), index=\'date\')\n .order_by(index=\'date\').run(conn);\n\n'), + (rethinkdb.ast.RqlQuery.sample, b"sequence.sample(number) -> selection\nstream.sample(number) -> array\narray.sample(number) -> array\n\nSelect a given number of elements from a sequence with uniform random distribution. Selection is done without replacement.\n\nIf the sequence has less than the requested number of elements (i.e., calling `sample(10)` on a sequence with only five elements), `sample` will return the entire sequence in a random order.\n\n*Example* Select 3 random heroes.\n\n r.table('marvel').sample(3).run(conn)\n"), + (rethinkdb.ast.RqlQuery.skip, b"sequence.skip(n) -> stream\narray.skip(n) -> array\n\nSkip a number of elements from the head of the sequence.\n\n*Example* Here in conjunction with `order_by` we choose to ignore the most successful heroes.\n\n r.table('marvel').order_by('successMetric').skip(10).run(conn)\n\n"), + (rethinkdb.ast.RqlQuery.slice, b"selection.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> selection\nstream.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> stream\narray.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> array\nbinary.slice(start_index[, end_index, left_bound='closed', right_bound='open']) -> binary\n\nReturn the elements of a sequence within the specified range.\n\n`slice` returns the range between `start_index` and `end_index`. If only `start_index` is specified, `slice` returns the range from that index to the end of the sequence. Specify `left_bound` or `right_bound` as `open` or `closed` to indicate whether to include that endpoint of the range by default: `closed` returns that endpoint, while `open` does not. By default, `left_bound` is closed and `right_bound` is open, so the range `(10,13)` will return the tenth, eleventh and twelfth elements in the sequence.\n\nIf `end_index` is past the end of the sequence, all elements from `start_index` to the end of the sequence will be returned. If `start_index` is past the end of the sequence or `end_index` is less than `start_index`, a zero-element sequence will be returned (although see below for negative `end_index` values). An error will be raised on a negative `start_index`.\n\nA negative `end_index` is allowed with arrays; in that case, the returned range counts backward from the array's end. That is, the range of `(2,-1)` returns the second element through the next-to-last element of the range. A negative `end_index` is not allowed with a stream. (An `end_index` of −1 *is* allowed with a stream if `right_bound` is closed; this behaves as if no `end_index` was specified.)\n\nIf `slice` is used with a [binary](http://rethinkdb.com/api/python/binary) object, the indexes refer to byte positions within the object. That is, the range `(10,20)` will refer to the 10th byte through the 19th byte.\n\nIf you are only specifying the indexes and not the bounding options, you may use Python's slice operator as a shorthand: `[start_index:end_index]`.\n\n**Example:** Return the fourth, fifth and sixth youngest players. (The youngest player is at index 0, so those are elements 3–5.)\n\n r.table('players').order_by(index='age').slice(3,6).run(conn)\n\nOr, using Python's slice operator:\n\n r.table('players').filter({'class': 'amateur'})[10:20].run(conn)\n\n**Example:** Return all but the top three players who have a red flag.\n\n r.table('players').filter({'flag': 'red'}).order_by(index=r.desc('score')).slice(3).run(conn)\n\n**Example:** Return holders of tickets `X` through `Y`, assuming tickets are numbered sequentially. We want to include ticket `Y`.\n\n r.table('users').order_by(index='ticket').slice(x, y, right_bound='closed').run(conn)\n\n**Example:** Return the elements of an array from the second through two from the end (that is, not including the last two).\n\n r.expr([0,1,2,3,4,5]).slice(2,-2).run(conn)\n\nResult:\n\n [2,3]\n"), + (rethinkdb.ast.RqlQuery.union, b"stream.union(sequence[, sequence, ...]) -> stream\narray.union(sequence[, sequence, ...]) -> array\n\nConcatenate two or more sequences.\n\n*Example* Construct a stream of all heroes.\n\n r.table('marvel').union(r.table('dc')).run(conn)\n\n*Example* Combine four arrays into one.\n\n r.expr([1, 2]).union([3, 4], [5, 6], [7, 8, 9]).run(conn)\n \n [1, 2, 3, 4, 5, 6, 7, 8, 9]\n"), + (rethinkdb.ast.RqlQuery.with_fields, b"sequence.with_fields([selector1, selector2...]) -> stream\narray.with_fields([selector1, selector2...]) -> array\n\nPlucks one or more attributes from a sequence of objects, filtering out any objects in the sequence that do not have the specified fields. Functionally, this is identical to `has_fields` followed by `pluck` on a sequence.\n\n*Example* Get a list of users and their posts, excluding any users who have not made any posts.\n\nExisting table structure:\n\n [\n { 'id': 1, 'user': 'bob', 'email': 'bob@foo.com', 'posts': [ 1, 4, 5 ] },\n { 'id': 2, 'user': 'george', 'email': 'george@foo.com' },\n { 'id': 3, 'user': 'jane', 'email': 'jane@foo.com', 'posts': [ 2, 3, 6 ] }\n ]\n\nCommand and output:\n\n r.table('users').with_fields('id', 'user', 'posts').run(conn)\n \n [\n { 'id': 1, 'user': 'bob', 'posts': [ 1, 4, 5 ] },\n { 'id': 3, 'user': 'jane', 'posts': [ 2, 3, 6 ] }\n ]\n\n*Example* Use the [nested field syntax](http://rethinkdb.com/docs/nested-fields/) to get a list of users with cell phone numbers in their contacts.\n\n r.table('users').with_fields('id', 'user', {contact: {'phone': 'work'}).run(conn)\n"), + (rethinkdb.ast.Table.delete, b'table.delete([durability="hard", return_changes=False])\n -> object\nselection.delete([durability="hard", return_changes=False])\n -> object\nsingleSelection.delete([durability="hard", return_changes=False])\n -> object\n\nDelete one or more documents from a table.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the\ntable or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). \nIn soft durability mode RethinkDB will acknowledge the write immediately after\nreceiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n\nDelete returns an object that contains the following attributes:\n\n- `deleted`: the number of documents that were deleted.\n- `skipped`: the number of documents that were skipped. \nFor example, if you attempt to delete a batch of documents, and another concurrent query\ndeletes some of those documents first, they will be counted as skipped.\n- `errors`: the number of errors encountered while performing the delete.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `inserted`, `replaced`, and `unchanged`: all 0 for a delete operation.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `delete` operation. Each object will have two keys: `{"new_val": None, "old_val": }`.\n\n*Example* Delete a single document from the table `comments`.\n\n r.table("comments").get("7eab9e63-73f1-4f33-8ce4-95cbea626f59").delete().run(conn)\n\n*Example* Delete all documents from the table `comments`.\n\n r.table("comments").delete().run(conn)\n\n*Example* Delete all comments where the field `id_post` is `3`.\n\n r.table("comments").filter({"id_post": 3}).delete().run(conn)\n\n*Example* Delete a single document from the table `comments` and return its value.\n\n r.table("comments").get("7eab9e63-73f1-4f33-8ce4-95cbea626f59").delete(return_changes=True).run(conn)\n\nThe result will look like:\n\n {\n "deleted": 1,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": None,\n "old_val": {\n "id": "7eab9e63-73f1-4f33-8ce4-95cbea626f59",\n "author": "William",\n "comment": "Great post",\n "id_post": 3\n }\n }\n ],\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\n*Example* Delete all documents from the table `comments` without waiting for the\noperation to be flushed to disk.\n\n r.table("comments").delete(durability="soft"}).run(conn)\n'), + (rethinkdb.ast.Table.insert, b'table.insert(object | [object1, object2, ...][, durability="hard", return_changes=False, conflict="error"])\n -> object\n\nInsert documents into a table. Accepts a single document or an array of\ndocuments.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the table or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). In soft durability mode RethinkDB will acknowledge the write immediately after receiving and caching it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `conflict`: Determine handling of inserting documents with the same primary key as existing entries. Possible values are `"error"`, `"replace"` or `"update"`.\n - `"error"`: Do not insert the new document and record the conflict as an error. This is the default.\n - `"replace"`: [Replace](http://rethinkdb.com/api/python/replace/) the old document in its entirety with the new one.\n - `"update"`: [Update](http://rethinkdb.com/api/python/update/) fields of the old document with fields from the new one.\n\nInsert returns an object that contains the following attributes:\n\n- `inserted`: the number of documents successfully inserted.\n- `replaced`: the number of documents updated when `conflict` is set to `"replace"` or `"update"`.\n- `unchanged`: the number of documents whose fields are identical to existing documents with the same primary key when `conflict` is set to `"replace"` or `"update"`.\n- `errors`: the number of errors encountered while performing the insert.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `deleted` and `skipped`: 0 for an insert operation.\n- `generated_keys`: a list of generated primary keys for inserted documents whose primary keys were not specified (capped to 100,000).\n- `warnings`: if the field `generated_keys` is truncated, you will get the warning _"Too many generated keys (<X>), array truncated to 100000."_.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `insert` operation. Each object will have two keys: `{"new_val": , "old_val": None}`.\n\n*Example* Insert a document into the table `posts`.\n\n r.table("posts").insert({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }).run(conn)\n\nThe result will be:\n\n {\n "deleted": 0,\n "errors": 0,\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\n*Example* Insert a document without a defined primary key into the table `posts` where the\nprimary key is `id`.\n\n r.table("posts").insert({\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }).run(conn)\n\nRethinkDB will generate a primary key and return it in `generated_keys`.\n\n {\n "deleted": 0,\n "errors": 0,\n "generated_keys": [\n "dd782b64-70a7-43e4-b65e-dd14ae61d947"\n ],\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\nRetrieve the document you just inserted with:\n\n r.table("posts").get("dd782b64-70a7-43e4-b65e-dd14ae61d947").run(conn)\n\nAnd you will get back:\n\n {\n "id": "dd782b64-70a7-43e4-b65e-dd14ae61d947",\n "title": "Lorem ipsum",\n "content": "Dolor sit amet",\n }\n\n*Example* Insert multiple documents into the table `users`.\n\n r.table("users").insert([\n {"id": "william", "email": "william@rethinkdb.com"},\n {"id": "lara", "email": "lara@rethinkdb.com"}\n ]).run(conn)\n\n*Example* Insert a document into the table `users`, replacing the document if the document\nalready exists. \n\n r.table("users").insert(\n {"id": "william", "email": "william@rethinkdb.com"},\n conflict="error"\n ).run(conn)\n\n*Example* Copy the documents from `posts` to `posts_backup`.\n\n r.table("posts_backup").insert( r.table("posts") ).run(conn)\n\n*Example* Get back a copy of the inserted document (with its generated primary key).\n\n r.table("posts").insert(\n {"title": "Lorem ipsum", "content": "Dolor sit amet"},\n return_changes=True\n ).run(conn)\n\nThe result will be\n\n {\n "deleted": 0,\n "errors": 0,\n "generated_keys": [\n "dd782b64-70a7-43e4-b65e-dd14ae61d947"\n ],\n "inserted": 1,\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0,\n "changes": [\n {\n "old_val": None,\n "new_val": {\n "id": "dd782b64-70a7-43e4-b65e-dd14ae61d947",\n "title": "Lorem ipsum",\n "content": "Dolor sit amet"\n }\n }\n ]\n }\n'), + (rethinkdb.ast.Table.replace, b'table.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nselection.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nsingleSelection.replace(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\n\nReplace documents in a table. Accepts a JSON document or a ReQL expression, and replaces\nthe original document with the new one. The new document must have the same primary key\nas the original document.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the\ntable or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). \nIn soft durability mode RethinkDB will acknowledge the write immediately after\nreceiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `non_atomic`: if set to `True`, executes the replacement and distributes the result to replicas in a non-atomic fashion. This flag is required to perform non-deterministic updates, such as those that require reading data from another table.\n\nReplace returns an object that contains the following attributes:\n\n- `replaced`: the number of documents that were replaced\n- `unchanged`: the number of documents that would have been modified, except that the\nnew value was the same as the old value\n- `inserted`: the number of new documents added. You can have new documents inserted if\nyou do a point-replace on a key that isn\'t in the table or you do a replace on a\nselection and one of the documents you are replacing has been deleted\n- `deleted`: the number of deleted documents when doing a replace with `None`\n- `errors`: the number of errors encountered while performing the replace.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `skipped`: 0 for a replace operation\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `replace` operation. Each object will have two keys: `{"new_val": , "old_val": }`.\n\n*Example* Replace the document with the primary key `1`.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "draft"\n }).run(conn)\n\n*Example* Remove the field `status` from all posts.\n\n r.table("posts").replace(lambda post:\n post.without("status")\n ).run(conn)\n\n*Example* Remove all the fields that are not `id`, `title` or `content`.\n\n r.table("posts").replace(lambda post:\n post.pluck("id", "title", "content")\n ).run(conn)\n\n*Example* Replace the document with the primary key `1` using soft durability.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "draft"\n }, durability="soft").run(conn)\n\n*Example* Replace the document with the primary key `1` and return the values of the document before\nand after the replace operation.\n\n r.table("posts").get(1).replace({\n "id": 1,\n "title": "Lorem ipsum",\n "content": "Aleas jacta est",\n "status": "published"\n }, return_changes=True).run(conn)\n\nThe result will have a `changes` field:\n\n {\n "deleted": 0,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": {\n "id":1,\n "title": "Lorem ipsum"\n "content": "Aleas jacta est",\n "status": "published",\n },\n "old_val": {\n "id":1,\n "title": "Lorem ipsum"\n "content": "TODO",\n "status": "draft",\n "author": "William",\n }\n }\n ], \n "replaced": 1,\n "skipped": 0,\n "unchanged": 0\n }\n'), + (rethinkdb.ast.Table.sync, b'table.sync() -> object\n\n`sync` ensures that writes on a given table are written to permanent storage. Queries\nthat specify soft durability (`durability=\'soft\'`) do not give such guarantees, so\n`sync` can be used to ensure the state of these queries. A call to `sync` does not return\nuntil all previous writes to the table are persisted.\n\nIf successful, the operation returns an object: `{"synced": 1}`.\n\n*Example* After having updated multiple heroes with soft durability, we now want to wait\nuntil these changes are persisted.\n\n r.table(\'marvel\').sync().run(conn)\n\n'), + (rethinkdb.ast.Table.update, b'table.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nselection.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\nsingleSelection.update(object | expr[, durability="hard", return_changes=False, non_atomic=False])\n -> object\n\nUpdate JSON documents in a table. Accepts a JSON document, a ReQL expression, or a combination of the two.\n\nThe optional arguments are:\n\n- `durability`: possible values are `hard` and `soft`. This option will override the table or query\'s durability setting (set in [run](http://rethinkdb.com/api/python/run/)). In soft durability mode RethinkDB will acknowledge the write immediately after receiving it, but before the write has been committed to disk.\n- `return_changes`:\n - `True`: return a `changes` array consisting of `old_val`/`new_val` objects describing the changes made, only including the documents actually updated.\n - `False`: do not return a `changes` array (the default).\n - `"always"`: behave as `True`, but include all documents the command tried to update whether or not the update was successful. (This was the behavior of `True` pre-2.0.)\n- `non_atomic`: if set to `True`, executes the update and distributes the result to replicas in a non-atomic fashion. This flag is required to perform non-deterministic updates, such as those that require reading data from another table.\n\nUpdate returns an object that contains the following attributes:\n\n- `replaced`: the number of documents that were updated.\n- `unchanged`: the number of documents that would have been modified except the new value was the same as the old value.\n- `skipped`: the number of documents that were skipped because the document didn\'t exist.\n- `errors`: the number of errors encountered while performing the update.\n- `first_error`: If errors were encountered, contains the text of the first error.\n- `deleted` and `inserted`: 0 for an update operation.\n- `changes`: if `return_changes` is set to `True`, this will be an array of objects, one for each objected affected by the `update` operation. Each object will have two keys: `{"new_val": , "old_val": }`.\n\n*Example* Update the status of the post with `id` of `1` to `published`.\n\n r.table("posts").get(1).update({"status": "published"}).run(conn)\n\n*Example* Update the status of all posts to `published`.\n\n r.table("posts").update({"status": "published"}).run(conn)\n\n*Example* Update the status of all the posts written by William.\n\n r.table("posts").filter({"author": "William"}).update({"status": "published"}).run(conn)\n\n*Example* Increment the field `view` with `id` of `1`.\nThis query will throw an error if the field `views` doesn\'t exist.\n\n r.table("posts").get(1).update({\n "views": r.row["views"]+1\n }).run(conn)\n\n*Example* Increment the field `view` of the post with `id` of `1`.\nIf the field `views` does not exist, it will be set to `0`.\n\n r.table("posts").update({\n "views": (r.row["views"]+1).default(0)\n }).run(conn)\n\n*Example* Perform a conditional update. \nIf the post has more than 100 views, set the `type` of a post to `hot`, else set it to `normal`.\n\n r.table("posts").get(1).update(lambda post:\n r.branch(\n post["views"] > 100,\n {"type": "hot"},\n {"type": "normal"}\n )\n ).run(conn)\n\n*Example* Update the field `num_comments` with the result of a sub-query. Because this update is not atomic, you must pass the `non_atomic` flag.\n\n r.table("posts").get(1).update({\n "num_comments": r.table("comments").filter({"id_post": 1}).count()\n }, non_atomic=True).run(conn)\n\nIf you forget to specify the `non_atomic` flag, you will get a `RqlRuntimeError`:\n\nRqlRuntimeError: Could not prove function deterministic. Maybe you want to use the non_atomic flag? \n\n*Example* Update the field `num_comments` with a random value between 0 and 100. This update cannot be proven deterministic because of `r.js` (and in fact is not), so you must pass the `non_atomic` flag.\n\n r.table("posts").get(1).update({\n "num_comments": r.js("Math.floor(Math.random()*100)")\n }, non_atomic=True).run(conn)\n\n*Example* Update the status of the post with `id` of `1` using soft durability.\n\n r.table("posts").get(1).update({status: "published"}, durability="soft").run(conn)\n\n*Example* Increment the field `views` and return the values of the document before and after the update operation.\n\n r.table("posts").get(1).update({\n "views": r.row["views"]+1\n }, return_changes=True).run(conn)\n\nThe result will now include a `changes` field:\n\n {\n "deleted": 1,\n "errors": 0,\n "inserted": 0,\n "changes": [\n {\n "new_val": {\n "id": 1,\n "author": "Julius_Caesar",\n "title": "Commentarii de Bello Gallico",\n "content": "Aleas jacta est",\n "views": 207\n },\n "old_val": {\n "id": 1,\n "author": "Julius_Caesar",\n "title": "Commentarii de Bello Gallico",\n "content": "Aleas jacta est",\n "views": 206\n }\n }\n ],\n "replaced": 0,\n "skipped": 0,\n "unchanged": 0\n }\n\nThe `update` command supports RethinkDB\'s [nested field][nf] syntax to update subdocuments. Consider a user table with contact information in this format:\n\n[nf]: /docs/nested-fields/python\n\n {\n "id": 10001,\n "name": "Bob Smith",\n "contact": {\n "phone": {\n "work": "408-555-1212",\n "home": "408-555-1213",\n "cell": "408-555-1214"\n },\n "email": {\n "work": "bob@smith.com",\n "home": "bobsmith@example.com",\n "other": "bobbys@moosecall.net"\n },\n "im": {\n "skype": "Bob Smith",\n "aim": "bobmoose",\n "icq": "nobodyremembersicqnumbers"\n }\n },\n "notes": [\n {\n "date": r.time(2014,1,1,\'Z\'),\n "from": "John Doe",\n "subject": "My name is even more boring than Bob\'s"\n },\n {\n "date": r.time(2014,2,2,\'Z\'),\n "from": "Bob Smith Sr",\n "subject": "Happy Second of February"\n }\n ]\n }\n\n*Example* Update Bob Smith\'s cell phone number.\n\n r.table("users").get(10001).update(\n {"contact": {"phone": {"cell": "408-555-4242"}}}\n ).run(conn)\n\n*Example* Add another note to Bob Smith\'s record.\n\n new_note = {\n "date": r.now(),\n "from": "Inigo Montoya",\n "subject": "You killed my father"\n }\n r.table("users").get(10001).update(\n {"notes": r.row["notes"].append(new_note)}\n ).run(conn)\n\n*Example* Send a note to every user with an ICQ number.\n\n icq_note = {\n "date": r.now(),\n "from": "Admin",\n "subject": "Welcome to the future"\n }\n r.table("users").filter(\n r.row.has_fields({"contact": {"im": "icq"}})\n ).update(\n {"notes": r.row["notes"].append(icq_note)}\n ).run(conn)\n\n*Example* Replace all of Bob\'s IM records. Normally, `update` will merge nested documents together; to replace the entire `"im"` document, use the [literal][] command.\n\n[literal]: /api/python/literal/\n\n r.table(\'users\').get(10001).update(\n {"contact": {"im": r.literal({"aim": "themoosemeister"})}}\n ).run(conn)\n'), +] + +for function, text in docsSource: + try: + text = str(text.decode('utf-8')) + except UnicodeEncodeError: + pass + if hasattr(function, "__func__"): + function.__func__.__doc__ = text + else: + function.__doc__ = text diff --git a/ext/librethinkdbxx/reql/ql2.proto b/ext/librethinkdbxx/reql/ql2.proto new file mode 100644 index 00000000..e40c5be5 --- /dev/null +++ b/ext/librethinkdbxx/reql/ql2.proto @@ -0,0 +1,843 @@ +//////////////////////////////////////////////////////////////////////////////// +// THE HIGH-LEVEL VIEW // +//////////////////////////////////////////////////////////////////////////////// + +// Process: When you first open a connection, send the magic number +// for the version of the protobuf you're targeting (in the [Version] +// enum). This should **NOT** be sent as a protobuf; just send the +// little-endian 32-bit integer over the wire raw. This number should +// only be sent once per connection. + +// The magic number shall be followed by an authorization key. The +// first 4 bytes are the length of the key to be sent as a little-endian +// 32-bit integer, followed by the key string. Even if there is no key, +// an empty string should be sent (length 0 and no data). + +// Following the authorization key, the client shall send a magic number +// for the communication protocol they want to use (in the [Protocol] +// enum). This shall be a little-endian 32-bit integer. + +// The server will then respond with a NULL-terminated string response. +// "SUCCESS" indicates that the connection has been accepted. Any other +// response indicates an error, and the response string should describe +// the error. + +// Next, for each query you want to send, construct a [Query] protobuf +// and serialize it to a binary blob. Send the blob's size to the +// server encoded as a little-endian 32-bit integer, followed by the +// blob itself. You will recieve a [Response] protobuf back preceded +// by its own size, once again encoded as a little-endian 32-bit +// integer. You can see an example exchange below in **EXAMPLE**. + +// A query consists of a [Term] to evaluate and a unique-per-connection +// [token]. + +// Tokens are used for two things: +// * Keeping track of which responses correspond to which queries. +// * Batched queries. Some queries return lots of results, so we send back +// batches of <1000, and you need to send a [CONTINUE] query with the same +// token to get more results from the original query. +//////////////////////////////////////////////////////////////////////////////// + +message VersionDummy { // We need to wrap it like this for some + // non-conforming protobuf libraries + // This enum contains the magic numbers for your version. See **THE HIGH-LEVEL + // VIEW** for what to do with it. + enum Version { + V0_1 = 0x3f61ba36; + V0_2 = 0x723081e1; // Authorization key during handshake + V0_3 = 0x5f75e83e; // Authorization key and protocol during handshake + V0_4 = 0x400c2d20; // Queries execute in parallel + V1_0 = 0x34c2bdc3; // Users and permissions + } + + // The protocol to use after the handshake, specified in V0_3 + enum Protocol { + PROTOBUF = 0x271ffc41; + JSON = 0x7e6970c7; + } +} + +// You send one of: +// * A [START] query with a [Term] to evaluate and a unique-per-connection token. +// * A [CONTINUE] query with the same token as a [START] query that returned +// [SUCCESS_PARTIAL] in its [Response]. +// * A [STOP] query with the same token as a [START] query that you want to stop. +// * A [NOREPLY_WAIT] query with a unique per-connection token. The server answers +// with a [WAIT_COMPLETE] [Response]. +// * A [SERVER_INFO] query. The server answers with a [SERVER_INFO] [Response]. +message Query { + enum QueryType { + START = 1; // Start a new query. + CONTINUE = 2; // Continue a query that returned [SUCCESS_PARTIAL] + // (see [Response]). + STOP = 3; // Stop a query partway through executing. + NOREPLY_WAIT = 4; // Wait for noreply operations to finish. + SERVER_INFO = 5; // Get server information. + } + optional QueryType type = 1; + // A [Term] is how we represent the operations we want a query to perform. + optional Term query = 2; // only present when [type] = [START] + optional int64 token = 3; + // This flag is ignored on the server. `noreply` should be added + // to `global_optargs` instead (the key "noreply" should map to + // either true or false). + optional bool OBSOLETE_noreply = 4 [default = false]; + + // If this is set to [true], then [Datum] values will sometimes be + // of [DatumType] [R_JSON] (see below). This can provide enormous + // speedups in languages with poor protobuf libraries. + optional bool accepts_r_json = 5 [default = false]; + + message AssocPair { + optional string key = 1; + optional Term val = 2; + } + repeated AssocPair global_optargs = 6; +} + +// A backtrace frame (see `backtrace` in Response below) +message Frame { + enum FrameType { + POS = 1; // Error occurred in a positional argument. + OPT = 2; // Error occurred in an optional argument. + } + optional FrameType type = 1; + optional int64 pos = 2; // The index of the positional argument. + optional string opt = 3; // The name of the optional argument. +} +message Backtrace { + repeated Frame frames = 1; +} + +// You get back a response with the same [token] as your query. +message Response { + enum ResponseType { + // These response types indicate success. + SUCCESS_ATOM = 1; // Query returned a single RQL datatype. + SUCCESS_SEQUENCE = 2; // Query returned a sequence of RQL datatypes. + SUCCESS_PARTIAL = 3; // Query returned a partial sequence of RQL + // datatypes. If you send a [CONTINUE] query with + // the same token as this response, you will get + // more of the sequence. Keep sending [CONTINUE] + // queries until you get back [SUCCESS_SEQUENCE]. + WAIT_COMPLETE = 4; // A [NOREPLY_WAIT] query completed. + SERVER_INFO = 5; // The data for a [SERVER_INFO] request. This is + // the same as `SUCCESS_ATOM` except that there will + // never be profiling data. + + // These response types indicate failure. + CLIENT_ERROR = 16; // Means the client is buggy. An example is if the + // client sends a malformed protobuf, or tries to + // send [CONTINUE] for an unknown token. + COMPILE_ERROR = 17; // Means the query failed during parsing or type + // checking. For example, if you pass too many + // arguments to a function. + RUNTIME_ERROR = 18; // Means the query failed at runtime. An example is + // if you add together two values from a table, but + // they turn out at runtime to be booleans rather + // than numbers. + } + optional ResponseType type = 1; + + // If `ResponseType` is `RUNTIME_ERROR`, this may be filled in with more + // information about the error. + enum ErrorType { + INTERNAL = 1000000; + RESOURCE_LIMIT = 2000000; + QUERY_LOGIC = 3000000; + NON_EXISTENCE = 3100000; + OP_FAILED = 4100000; + OP_INDETERMINATE = 4200000; + USER = 5000000; + PERMISSION_ERROR = 6000000; + } + optional ErrorType error_type = 7; + + // ResponseNotes are used to provide information about the query + // response that may be useful for people writing drivers or ORMs. + // Currently all the notes we send indicate that a stream has certain + // special properties. + enum ResponseNote { + // The stream is a changefeed stream (e.g. `r.table('test').changes()`). + SEQUENCE_FEED = 1; + // The stream is a point changefeed stream + // (e.g. `r.table('test').get(0).changes()`). + ATOM_FEED = 2; + // The stream is an order_by_limit changefeed stream + // (e.g. `r.table('test').order_by(index: 'id').limit(5).changes()`). + ORDER_BY_LIMIT_FEED = 3; + // The stream is a union of multiple changefeed types that can't be + // collapsed to a single type + // (e.g. `r.table('test').changes().union(r.table('test').get(0).changes())`). + UNIONED_FEED = 4; + // The stream is a changefeed stream and includes notes on what state + // the changefeed stream is in (e.g. objects of the form `{state: + // 'initializing'}`). + INCLUDES_STATES = 5; + } + repeated ResponseNote notes = 6; + + optional int64 token = 2; // Indicates what [Query] this response corresponds to. + + // [response] contains 1 RQL datum if [type] is [SUCCESS_ATOM] or + // [SERVER_INFO]. [response] contains many RQL data if [type] is + // [SUCCESS_SEQUENCE] or [SUCCESS_PARTIAL]. [response] contains 1 + // error message (of type [R_STR]) in all other cases. + repeated Datum response = 3; + + // If [type] is [CLIENT_ERROR], [TYPE_ERROR], or [RUNTIME_ERROR], then a + // backtrace will be provided. The backtrace says where in the query the + // error occurred. Ideally this information will be presented to the user as + // a pretty-printed version of their query with the erroneous section + // underlined. A backtrace is a series of 0 or more [Frame]s, each of which + // specifies either the index of a positional argument or the name of an + // optional argument. (Those words will make more sense if you look at the + // [Term] message below.) + optional Backtrace backtrace = 4; // Contains n [Frame]s when you get back an error. + + // If the [global_optargs] in the [Query] that this [Response] is a + // response to contains a key "profile" which maps to a static value of + // true then [profile] will contain a [Datum] which provides profiling + // information about the execution of the query. This field should be + // returned to the user along with the result that would normally be + // returned (a datum or a cursor). In official drivers this is accomplished + // by putting them inside of an object with "value" mapping to the return + // value and "profile" mapping to the profile object. + optional Datum profile = 5; +} + +// A [Datum] is a chunk of data that can be serialized to disk or returned to +// the user in a Response. Currently we only support JSON types, but we may +// support other types in the future (e.g., a date type or an integer type). +message Datum { + enum DatumType { + R_NULL = 1; + R_BOOL = 2; + R_NUM = 3; // a double + R_STR = 4; + R_ARRAY = 5; + R_OBJECT = 6; + // This [DatumType] will only be used if [accepts_r_json] is + // set to [true] in [Query]. [r_str] will be filled with a + // JSON encoding of the [Datum]. + R_JSON = 7; // uses r_str + } + optional DatumType type = 1; + optional bool r_bool = 2; + optional double r_num = 3; + optional string r_str = 4; + + repeated Datum r_array = 5; + message AssocPair { + optional string key = 1; + optional Datum val = 2; + } + repeated AssocPair r_object = 6; +} + +// A [Term] is either a piece of data (see **Datum** above), or an operator and +// its operands. If you have a [Datum], it's stored in the member [datum]. If +// you have an operator, its positional arguments are stored in [args] and its +// optional arguments are stored in [optargs]. +// +// A note about type signatures: +// We use the following notation to denote types: +// arg1_type, arg2_type, argrest_type... -> result_type +// So, for example, if we have a function `avg` that takes any number of +// arguments and averages them, we might write: +// NUMBER... -> NUMBER +// Or if we had a function that took one number modulo another: +// NUMBER, NUMBER -> NUMBER +// Or a function that takes a table and a primary key of any Datum type, then +// retrieves the entry with that primary key: +// Table, DATUM -> OBJECT +// Some arguments must be provided as literal values (and not the results of sub +// terms). These are marked with a `!`. +// Optional arguments are specified within curly braces as argname `:` value +// type (e.x `{noreply:BOOL}`) +// Many RQL operations are polymorphic. For these, alterantive type signatures +// are separated by `|`. +// +// The RQL type hierarchy is as follows: +// Top +// DATUM +// NULL +// BOOL +// NUMBER +// STRING +// OBJECT +// SingleSelection +// ARRAY +// Sequence +// ARRAY +// Stream +// StreamSelection +// Table +// Database +// Function +// Ordering - used only by ORDER_BY +// Pathspec -- an object, string, or array that specifies a path +// Error +message Term { + enum TermType { + // A RQL datum, stored in `datum` below. + DATUM = 1; + + MAKE_ARRAY = 2; // DATUM... -> ARRAY + // Evaluate the terms in [optargs] and make an object + MAKE_OBJ = 3; // {...} -> OBJECT + + // * Compound types + + // Takes an integer representing a variable and returns the value stored + // in that variable. It's the responsibility of the client to translate + // from their local representation of a variable to a unique _non-negative_ + // integer for that variable. (We do it this way instead of letting + // clients provide variable names as strings to discourage + // variable-capturing client libraries, and because it's more efficient + // on the wire.) + VAR = 10; // !NUMBER -> DATUM + // Takes some javascript code and executes it. + JAVASCRIPT = 11; // STRING {timeout: !NUMBER} -> DATUM | + // STRING {timeout: !NUMBER} -> Function(*) + UUID = 169; // () -> DATUM + + // Takes an HTTP URL and gets it. If the get succeeds and + // returns valid JSON, it is converted into a DATUM + HTTP = 153; // STRING {data: OBJECT | STRING, + // timeout: !NUMBER, + // method: STRING, + // params: OBJECT, + // header: OBJECT | ARRAY, + // attempts: NUMBER, + // redirects: NUMBER, + // verify: BOOL, + // page: FUNC | STRING, + // page_limit: NUMBER, + // auth: OBJECT, + // result_format: STRING, + // } -> STRING | STREAM + + // Takes a string and throws an error with that message. + // Inside of a `default` block, you can omit the first + // argument to rethrow whatever error you catch (this is most + // useful as an argument to the `default` filter optarg). + ERROR = 12; // STRING -> Error | -> Error + // Takes nothing and returns a reference to the implicit variable. + IMPLICIT_VAR = 13; // -> DATUM + + // * Data Operators + // Returns a reference to a database. + DB = 14; // STRING -> Database + // Returns a reference to a table. + TABLE = 15; // Database, STRING, {read_mode:STRING, identifier_format:STRING} -> Table + // STRING, {read_mode:STRING, identifier_format:STRING} -> Table + // Gets a single element from a table by its primary or a secondary key. + GET = 16; // Table, STRING -> SingleSelection | Table, NUMBER -> SingleSelection | + // Table, STRING -> NULL | Table, NUMBER -> NULL | + GET_ALL = 78; // Table, DATUM..., {index:!STRING} => ARRAY + + // Simple DATUM Ops + EQ = 17; // DATUM... -> BOOL + NE = 18; // DATUM... -> BOOL + LT = 19; // DATUM... -> BOOL + LE = 20; // DATUM... -> BOOL + GT = 21; // DATUM... -> BOOL + GE = 22; // DATUM... -> BOOL + NOT = 23; // BOOL -> BOOL + // ADD can either add two numbers or concatenate two arrays. + ADD = 24; // NUMBER... -> NUMBER | STRING... -> STRING + SUB = 25; // NUMBER... -> NUMBER + MUL = 26; // NUMBER... -> NUMBER + DIV = 27; // NUMBER... -> NUMBER + MOD = 28; // NUMBER, NUMBER -> NUMBER + + FLOOR = 183; // NUMBER -> NUMBER + CEIL = 184; // NUMBER -> NUMBER + ROUND = 185; // NUMBER -> NUMBER + + // DATUM Array Ops + // Append a single element to the end of an array (like `snoc`). + APPEND = 29; // ARRAY, DATUM -> ARRAY + // Prepend a single element to the end of an array (like `cons`). + PREPEND = 80; // ARRAY, DATUM -> ARRAY + //Remove the elements of one array from another array. + DIFFERENCE = 95; // ARRAY, ARRAY -> ARRAY + + // DATUM Set Ops + // Set ops work on arrays. They don't use actual sets and thus have + // performance characteristics you would expect from arrays rather than + // from sets. All set operations have the post condition that they + // array they return contains no duplicate values. + SET_INSERT = 88; // ARRAY, DATUM -> ARRAY + SET_INTERSECTION = 89; // ARRAY, ARRAY -> ARRAY + SET_UNION = 90; // ARRAY, ARRAY -> ARRAY + SET_DIFFERENCE = 91; // ARRAY, ARRAY -> ARRAY + + SLICE = 30; // Sequence, NUMBER, NUMBER -> Sequence + SKIP = 70; // Sequence, NUMBER -> Sequence + LIMIT = 71; // Sequence, NUMBER -> Sequence + OFFSETS_OF = 87; // Sequence, DATUM -> Sequence | Sequence, Function(1) -> Sequence + CONTAINS = 93; // Sequence, (DATUM | Function(1))... -> BOOL + + // Stream/Object Ops + // Get a particular field from an object, or map that over a + // sequence. + GET_FIELD = 31; // OBJECT, STRING -> DATUM + // | Sequence, STRING -> Sequence + // Return an array containing the keys of the object. + KEYS = 94; // OBJECT -> ARRAY + // Return an array containing the values of the object. + VALUES = 186; // OBJECT -> ARRAY + // Creates an object + OBJECT = 143; // STRING, DATUM, ... -> OBJECT + // Check whether an object contains all the specified fields, + // or filters a sequence so that all objects inside of it + // contain all the specified fields. + HAS_FIELDS = 32; // OBJECT, Pathspec... -> BOOL + // x.with_fields(...) <=> x.has_fields(...).pluck(...) + WITH_FIELDS = 96; // Sequence, Pathspec... -> Sequence + // Get a subset of an object by selecting some attributes to preserve, + // or map that over a sequence. (Both pick and pluck, polymorphic.) + PLUCK = 33; // Sequence, Pathspec... -> Sequence | OBJECT, Pathspec... -> OBJECT + // Get a subset of an object by selecting some attributes to discard, or + // map that over a sequence. (Both unpick and without, polymorphic.) + WITHOUT = 34; // Sequence, Pathspec... -> Sequence | OBJECT, Pathspec... -> OBJECT + // Merge objects (right-preferential) + MERGE = 35; // OBJECT... -> OBJECT | Sequence -> Sequence + + // Sequence Ops + // Get all elements of a sequence between two values. + // Half-open by default, but the openness of either side can be + // changed by passing 'closed' or 'open for `right_bound` or + // `left_bound`. + BETWEEN_DEPRECATED = 36; // Deprecated version of between, which allows `null` to specify unboundedness + // With the newer version, clients should use `r.minval` and `r.maxval` for unboundedness + BETWEEN = 182; // StreamSelection, DATUM, DATUM, {index:!STRING, right_bound:STRING, left_bound:STRING} -> StreamSelection + REDUCE = 37; // Sequence, Function(2) -> DATUM + MAP = 38; // Sequence, Function(1) -> Sequence + // The arity of the function should be + // Sequence..., Function(sizeof...(Sequence)) -> Sequence + + FOLD = 187; // Sequence, Datum, Function(2), {Function(3), Function(1) + + // Filter a sequence with either a function or a shortcut + // object (see API docs for details). The body of FILTER is + // wrapped in an implicit `.default(false)`, and you can + // change the default value by specifying the `default` + // optarg. If you make the default `r.error`, all errors + // caught by `default` will be rethrown as if the `default` + // did not exist. + FILTER = 39; // Sequence, Function(1), {default:DATUM} -> Sequence | + // Sequence, OBJECT, {default:DATUM} -> Sequence + // Map a function over a sequence and then concatenate the results together. + CONCAT_MAP = 40; // Sequence, Function(1) -> Sequence + // Order a sequence based on one or more attributes. + ORDER_BY = 41; // Sequence, (!STRING | Ordering)..., {index: (!STRING | Ordering)} -> Sequence + // Get all distinct elements of a sequence (like `uniq`). + DISTINCT = 42; // Sequence -> Sequence + // Count the number of elements in a sequence, or only the elements that match + // a given filter. + COUNT = 43; // Sequence -> NUMBER | Sequence, DATUM -> NUMBER | Sequence, Function(1) -> NUMBER + IS_EMPTY = 86; // Sequence -> BOOL + // Take the union of multiple sequences (preserves duplicate elements! (use distinct)). + UNION = 44; // Sequence... -> Sequence + // Get the Nth element of a sequence. + NTH = 45; // Sequence, NUMBER -> DATUM + // do NTH or GET_FIELD depending on target object + BRACKET = 170; // Sequence | OBJECT, NUMBER | STRING -> DATUM + // OBSOLETE_GROUPED_MAPREDUCE = 46; + // OBSOLETE_GROUPBY = 47; + + INNER_JOIN = 48; // Sequence, Sequence, Function(2) -> Sequence + OUTER_JOIN = 49; // Sequence, Sequence, Function(2) -> Sequence + // An inner-join that does an equality comparison on two attributes. + EQ_JOIN = 50; // Sequence, !STRING, Sequence, {index:!STRING} -> Sequence + ZIP = 72; // Sequence -> Sequence + RANGE = 173; // -> Sequence [0, +inf) + // NUMBER -> Sequence [0, a) + // NUMBER, NUMBER -> Sequence [a, b) + + // Array Ops + // Insert an element in to an array at a given index. + INSERT_AT = 82; // ARRAY, NUMBER, DATUM -> ARRAY + // Remove an element at a given index from an array. + DELETE_AT = 83; // ARRAY, NUMBER -> ARRAY | + // ARRAY, NUMBER, NUMBER -> ARRAY + // Change the element at a given index of an array. + CHANGE_AT = 84; // ARRAY, NUMBER, DATUM -> ARRAY + // Splice one array in to another array. + SPLICE_AT = 85; // ARRAY, NUMBER, ARRAY -> ARRAY + + // * Type Ops + // Coerces a datum to a named type (e.g. "bool"). + // If you previously used `stream_to_array`, you should use this instead + // with the type "array". + COERCE_TO = 51; // Top, STRING -> Top + // Returns the named type of a datum (e.g. TYPE_OF(true) = "BOOL") + TYPE_OF = 52; // Top -> STRING + + // * Write Ops (the OBJECTs contain data about number of errors etc.) + // Updates all the rows in a selection. Calls its Function with the row + // to be updated, and then merges the result of that call. + UPDATE = 53; // StreamSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | + // SingleSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | + // StreamSelection, OBJECT, {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | + // SingleSelection, OBJECT, {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT + // Deletes all the rows in a selection. + DELETE = 54; // StreamSelection, {durability:STRING, return_changes:BOOL} -> OBJECT | SingleSelection -> OBJECT + // Replaces all the rows in a selection. Calls its Function with the row + // to be replaced, and then discards it and stores the result of that + // call. + REPLACE = 55; // StreamSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT | SingleSelection, Function(1), {non_atomic:BOOL, durability:STRING, return_changes:BOOL} -> OBJECT + // Inserts into a table. If `conflict` is replace, overwrites + // entries with the same primary key. If `conflict` is + // update, does an update on the entry. If `conflict` is + // error, or is omitted, conflicts will trigger an error. + INSERT = 56; // Table, OBJECT, {conflict:STRING, durability:STRING, return_changes:BOOL} -> OBJECT | Table, Sequence, {conflict:STRING, durability:STRING, return_changes:BOOL} -> OBJECT + + // * Administrative OPs + // Creates a database with a particular name. + DB_CREATE = 57; // STRING -> OBJECT + // Drops a database with a particular name. + DB_DROP = 58; // STRING -> OBJECT + // Lists all the databases by name. (Takes no arguments) + DB_LIST = 59; // -> ARRAY + // Creates a table with a particular name in a particular + // database. (You may omit the first argument to use the + // default database.) + TABLE_CREATE = 60; // Database, STRING, {primary_key:STRING, shards:NUMBER, replicas:NUMBER, primary_replica_tag:STRING} -> OBJECT + // Database, STRING, {primary_key:STRING, shards:NUMBER, replicas:OBJECT, primary_replica_tag:STRING} -> OBJECT + // STRING, {primary_key:STRING, shards:NUMBER, replicas:NUMBER, primary_replica_tag:STRING} -> OBJECT + // STRING, {primary_key:STRING, shards:NUMBER, replicas:OBJECT, primary_replica_tag:STRING} -> OBJECT + // Drops a table with a particular name from a particular + // database. (You may omit the first argument to use the + // default database.) + TABLE_DROP = 61; // Database, STRING -> OBJECT + // STRING -> OBJECT + // Lists all the tables in a particular database. (You may + // omit the first argument to use the default database.) + TABLE_LIST = 62; // Database -> ARRAY + // -> ARRAY + // Returns the row in the `rethinkdb.table_config` or `rethinkdb.db_config` table + // that corresponds to the given database or table. + CONFIG = 174; // Database -> SingleSelection + // Table -> SingleSelection + // Returns the row in the `rethinkdb.table_status` table that corresponds to the + // given table. + STATUS = 175; // Table -> SingleSelection + // Called on a table, waits for that table to be ready for read/write operations. + // Called on a database, waits for all of the tables in the database to be ready. + // Returns the corresponding row or rows from the `rethinkdb.table_status` table. + WAIT = 177; // Table -> OBJECT + // Database -> OBJECT + // Generates a new config for the given table, or all tables in the given database + // The `shards` and `replicas` arguments are required. If `emergency_repair` is + // specified, it will enter a completely different mode of repairing a table + // which has lost half or more of its replicas. + RECONFIGURE = 176; // Database|Table, {shards:NUMBER, replicas:NUMBER [, + // dry_run:BOOLEAN] + // } -> OBJECT + // Database|Table, {shards:NUMBER, replicas:OBJECT [, + // primary_replica_tag:STRING, + // nonvoting_replica_tags:ARRAY, + // dry_run:BOOLEAN] + // } -> OBJECT + // Table, {emergency_repair:STRING, dry_run:BOOLEAN} -> OBJECT + // Balances the table's shards but leaves everything else the same. Can also be + // applied to an entire database at once. + REBALANCE = 179; // Table -> OBJECT + // Database -> OBJECT + + // Ensures that previously issued soft-durability writes are complete and + // written to disk. + SYNC = 138; // Table -> OBJECT + + // Set global, database, or table-specific permissions + GRANT = 188; // -> OBJECT + // Database -> OBJECT + // Table -> OBJECT + + // * Secondary indexes OPs + // Creates a new secondary index with a particular name and definition. + INDEX_CREATE = 75; // Table, STRING, Function(1), {multi:BOOL} -> OBJECT + // Drops a secondary index with a particular name from the specified table. + INDEX_DROP = 76; // Table, STRING -> OBJECT + // Lists all secondary indexes on a particular table. + INDEX_LIST = 77; // Table -> ARRAY + // Gets information about whether or not a set of indexes are ready to + // be accessed. Returns a list of objects that look like this: + // {index:STRING, ready:BOOL[, progress:NUMBER]} + INDEX_STATUS = 139; // Table, STRING... -> ARRAY + // Blocks until a set of indexes are ready to be accessed. Returns the + // same values INDEX_STATUS. + INDEX_WAIT = 140; // Table, STRING... -> ARRAY + // Renames the given index to a new name + INDEX_RENAME = 156; // Table, STRING, STRING, {overwrite:BOOL} -> OBJECT + + // * Control Operators + // Calls a function on data + FUNCALL = 64; // Function(*), DATUM... -> DATUM + // Executes its first argument, and returns its second argument if it + // got [true] or its third argument if it got [false] (like an `if` + // statement). + BRANCH = 65; // BOOL, Top, Top -> Top + // Returns true if any of its arguments returns true (short-circuits). + OR = 66; // BOOL... -> BOOL + // Returns true if all of its arguments return true (short-circuits). + AND = 67; // BOOL... -> BOOL + // Calls its Function with each entry in the sequence + // and executes the array of terms that Function returns. + FOR_EACH = 68; // Sequence, Function(1) -> OBJECT + +//////////////////////////////////////////////////////////////////////////////// +////////// Special Terms +//////////////////////////////////////////////////////////////////////////////// + + // An anonymous function. Takes an array of numbers representing + // variables (see [VAR] above), and a [Term] to execute with those in + // scope. Returns a function that may be passed an array of arguments, + // then executes the Term with those bound to the variable names. The + // user will never construct this directly. We use it internally for + // things like `map` which take a function. The "arity" of a [Function] is + // the number of arguments it takes. + // For example, here's what `_X_.map{|x| x+2}` turns into: + // Term { + // type = MAP; + // args = [_X_, + // Term { + // type = Function; + // args = [Term { + // type = DATUM; + // datum = Datum { + // type = R_ARRAY; + // r_array = [Datum { type = R_NUM; r_num = 1; }]; + // }; + // }, + // Term { + // type = ADD; + // args = [Term { + // type = VAR; + // args = [Term { + // type = DATUM; + // datum = Datum { type = R_NUM; + // r_num = 1}; + // }]; + // }, + // Term { + // type = DATUM; + // datum = Datum { type = R_NUM; r_num = 2; }; + // }]; + // }]; + // }]; + FUNC = 69; // ARRAY, Top -> ARRAY -> Top + + // Indicates to ORDER_BY that this attribute is to be sorted in ascending order. + ASC = 73; // !STRING -> Ordering + // Indicates to ORDER_BY that this attribute is to be sorted in descending order. + DESC = 74; // !STRING -> Ordering + + // Gets info about anything. INFO is most commonly called on tables. + INFO = 79; // Top -> OBJECT + + // `a.match(b)` returns a match object if the string `a` + // matches the regular expression `b`. + MATCH = 97; // STRING, STRING -> DATUM + + // Change the case of a string. + UPCASE = 141; // STRING -> STRING + DOWNCASE = 142; // STRING -> STRING + + // Select a number of elements from sequence with uniform distribution. + SAMPLE = 81; // Sequence, NUMBER -> Sequence + + // Evaluates its first argument. If that argument returns + // NULL or throws an error related to the absence of an + // expected value (for instance, accessing a non-existent + // field or adding NULL to an integer), DEFAULT will either + // return its second argument or execute it if it's a + // function. If the second argument is a function, it will be + // passed either the text of the error or NULL as its + // argument. + DEFAULT = 92; // Top, Top -> Top + + // Parses its first argument as a json string and returns it as a + // datum. + JSON = 98; // STRING -> DATUM + // Returns the datum as a JSON string. + // N.B.: we would really prefer this be named TO_JSON and that exists as + // an alias in Python and JavaScript drivers; however it conflicts with the + // standard `to_json` method defined by Ruby's standard json library. + TO_JSON_STRING = 172; // DATUM -> STRING + + // Parses its first arguments as an ISO 8601 time and returns it as a + // datum. + ISO8601 = 99; // STRING -> PSEUDOTYPE(TIME) + // Prints a time as an ISO 8601 time. + TO_ISO8601 = 100; // PSEUDOTYPE(TIME) -> STRING + + // Returns a time given seconds since epoch in UTC. + EPOCH_TIME = 101; // NUMBER -> PSEUDOTYPE(TIME) + // Returns seconds since epoch in UTC given a time. + TO_EPOCH_TIME = 102; // PSEUDOTYPE(TIME) -> NUMBER + + // The time the query was received by the server. + NOW = 103; // -> PSEUDOTYPE(TIME) + // Puts a time into an ISO 8601 timezone. + IN_TIMEZONE = 104; // PSEUDOTYPE(TIME), STRING -> PSEUDOTYPE(TIME) + // a.during(b, c) returns whether a is in the range [b, c) + DURING = 105; // PSEUDOTYPE(TIME), PSEUDOTYPE(TIME), PSEUDOTYPE(TIME) -> BOOL + // Retrieves the date portion of a time. + DATE = 106; // PSEUDOTYPE(TIME) -> PSEUDOTYPE(TIME) + // x.time_of_day == x.date - x + TIME_OF_DAY = 126; // PSEUDOTYPE(TIME) -> NUMBER + // Returns the timezone of a time. + TIMEZONE = 127; // PSEUDOTYPE(TIME) -> STRING + + // These access the various components of a time. + YEAR = 128; // PSEUDOTYPE(TIME) -> NUMBER + MONTH = 129; // PSEUDOTYPE(TIME) -> NUMBER + DAY = 130; // PSEUDOTYPE(TIME) -> NUMBER + DAY_OF_WEEK = 131; // PSEUDOTYPE(TIME) -> NUMBER + DAY_OF_YEAR = 132; // PSEUDOTYPE(TIME) -> NUMBER + HOURS = 133; // PSEUDOTYPE(TIME) -> NUMBER + MINUTES = 134; // PSEUDOTYPE(TIME) -> NUMBER + SECONDS = 135; // PSEUDOTYPE(TIME) -> NUMBER + + // Construct a time from a date and optional timezone or a + // date+time and optional timezone. + TIME = 136; // NUMBER, NUMBER, NUMBER, STRING -> PSEUDOTYPE(TIME) | + // NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, STRING -> PSEUDOTYPE(TIME) | + + // Constants for ISO 8601 days of the week. + MONDAY = 107; // -> 1 + TUESDAY = 108; // -> 2 + WEDNESDAY = 109; // -> 3 + THURSDAY = 110; // -> 4 + FRIDAY = 111; // -> 5 + SATURDAY = 112; // -> 6 + SUNDAY = 113; // -> 7 + + // Constants for ISO 8601 months. + JANUARY = 114; // -> 1 + FEBRUARY = 115; // -> 2 + MARCH = 116; // -> 3 + APRIL = 117; // -> 4 + MAY = 118; // -> 5 + JUNE = 119; // -> 6 + JULY = 120; // -> 7 + AUGUST = 121; // -> 8 + SEPTEMBER = 122; // -> 9 + OCTOBER = 123; // -> 10 + NOVEMBER = 124; // -> 11 + DECEMBER = 125; // -> 12 + + // Indicates to MERGE to replace, or remove in case of an empty literal, the + // other object rather than merge it. + LITERAL = 137; // -> Merging + // JSON -> Merging + + // SEQUENCE, STRING -> GROUPED_SEQUENCE | SEQUENCE, FUNCTION -> GROUPED_SEQUENCE + GROUP = 144; + SUM = 145; + AVG = 146; + MIN = 147; + MAX = 148; + + // `str.split()` splits on whitespace + // `str.split(" ")` splits on spaces only + // `str.split(" ", 5)` splits on spaces with at most 5 results + // `str.split(nil, 5)` splits on whitespace with at most 5 results + SPLIT = 149; // STRING -> ARRAY | STRING, STRING -> ARRAY | STRING, STRING, NUMBER -> ARRAY | STRING, NULL, NUMBER -> ARRAY + + UNGROUP = 150; // GROUPED_DATA -> ARRAY + + // Takes a range of numbers and returns a random number within the range + RANDOM = 151; // NUMBER, NUMBER {float:BOOL} -> DATUM + + CHANGES = 152; // TABLE -> STREAM + ARGS = 154; // ARRAY -> SPECIAL (used to splice arguments) + + // BINARY is client-only at the moment, it is not supported on the server + BINARY = 155; // STRING -> PSEUDOTYPE(BINARY) + + GEOJSON = 157; // OBJECT -> PSEUDOTYPE(GEOMETRY) + TO_GEOJSON = 158; // PSEUDOTYPE(GEOMETRY) -> OBJECT + POINT = 159; // NUMBER, NUMBER -> PSEUDOTYPE(GEOMETRY) + LINE = 160; // (ARRAY | PSEUDOTYPE(GEOMETRY))... -> PSEUDOTYPE(GEOMETRY) + POLYGON = 161; // (ARRAY | PSEUDOTYPE(GEOMETRY))... -> PSEUDOTYPE(GEOMETRY) + DISTANCE = 162; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) {geo_system:STRING, unit:STRING} -> NUMBER + INTERSECTS = 163; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> BOOL + INCLUDES = 164; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> BOOL + CIRCLE = 165; // PSEUDOTYPE(GEOMETRY), NUMBER {num_vertices:NUMBER, geo_system:STRING, unit:STRING, fill:BOOL} -> PSEUDOTYPE(GEOMETRY) + GET_INTERSECTING = 166; // TABLE, PSEUDOTYPE(GEOMETRY) {index:!STRING} -> StreamSelection + FILL = 167; // PSEUDOTYPE(GEOMETRY) -> PSEUDOTYPE(GEOMETRY) + GET_NEAREST = 168; // TABLE, PSEUDOTYPE(GEOMETRY) {index:!STRING, max_results:NUM, max_dist:NUM, geo_system:STRING, unit:STRING} -> ARRAY + POLYGON_SUB = 171; // PSEUDOTYPE(GEOMETRY), PSEUDOTYPE(GEOMETRY) -> PSEUDOTYPE(GEOMETRY) + + // Constants for specifying key ranges + MINVAL = 180; + MAXVAL = 181; + } + optional TermType type = 1; + + // This is only used when type is DATUM. + optional Datum datum = 2; + + repeated Term args = 3; // Holds the positional arguments of the query. + message AssocPair { + optional string key = 1; + optional Term val = 2; + } + repeated AssocPair optargs = 4; // Holds the optional arguments of the query. + // (Note that the order of the optional arguments doesn't matter; think of a + // Hash.) +} + +//////////////////////////////////////////////////////////////////////////////// +// EXAMPLE // +//////////////////////////////////////////////////////////////////////////////// +// ```ruby +// r.table('tbl', {:read_mode => 'outdated'}).insert([{:id => 0}, {:id => 1}]) +// ``` +// Would turn into: +// Term { +// type = INSERT; +// args = [Term { +// type = TABLE; +// args = [Term { +// type = DATUM; +// datum = Datum { type = R_STR; r_str = "tbl"; }; +// }]; +// optargs = [["read_mode", +// Term { +// type = DATUM; +// datum = Datum { type = R_STR; r_bool = "outdated"; }; +// }]]; +// }, +// Term { +// type = MAKE_ARRAY; +// args = [Term { +// type = DATUM; +// datum = Datum { type = R_OBJECT; r_object = [["id", 0]]; }; +// }, +// Term { +// type = DATUM; +// datum = Datum { type = R_OBJECT; r_object = [["id", 1]]; }; +// }]; +// }] +// } +// And the server would reply: +// Response { +// type = SUCCESS_ATOM; +// token = 1; +// response = [Datum { type = R_OBJECT; r_object = [["inserted", 2]]; }]; +// } +// Or, if there were an error: +// Response { +// type = RUNTIME_ERROR; +// token = 1; +// response = [Datum { type = R_STR; r_str = "The table `tbl` doesn't exist!"; }]; +// backtrace = [Frame { type = POS; pos = 0; }, Frame { type = POS; pos = 0; }]; +// } diff --git a/ext/librethinkdbxx/src/connection.cc b/ext/librethinkdbxx/src/connection.cc new file mode 100644 index 00000000..62f6efee --- /dev/null +++ b/ext/librethinkdbxx/src/connection.cc @@ -0,0 +1,431 @@ +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include "connection.h" +#include "connection_p.h" +#include "json_p.h" +#include "exceptions.h" +#include "term.h" +#include "cursor_p.h" + +#include "rapidjson-config.h" +#include "rapidjson/rapidjson.h" +#include "rapidjson/encodedstream.h" +#include "rapidjson/document.h" + +namespace RethinkDB { + +using QueryType = Protocol::Query::QueryType; + +// constants +const int debug_net = 0; +const uint32_t version_magic = + static_cast(Protocol::VersionDummy::Version::V0_4); +const uint32_t json_magic = + static_cast(Protocol::VersionDummy::Protocol::JSON); + +std::unique_ptr connect(std::string host, int port, std::string auth_key) { + struct addrinfo hints; + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + char port_str[16]; + snprintf(port_str, 16, "%d", port); + struct addrinfo *servinfo; + int ret = getaddrinfo(host.c_str(), port_str, &hints, &servinfo); + if (ret) throw Error("getaddrinfo: %s\n", gai_strerror(ret)); + + struct addrinfo *p; + Error error; + int sockfd; + for (p = servinfo; p != NULL; p = p->ai_next) { + sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); + if (sockfd == -1) { + error = Error::from_errno("socket"); + continue; + } + + if (connect(sockfd, p->ai_addr, p->ai_addrlen) == -1) { + ::close(sockfd); + error = Error::from_errno("connect"); + continue; + } + + break; + } + + if (p == NULL) { + throw error; + } + + freeaddrinfo(servinfo); + + std::unique_ptr conn_private(new ConnectionPrivate(sockfd)); + WriteLock writer(conn_private.get()); + { + size_t size = auth_key.size(); + char buf[12 + size]; + memcpy(buf, &version_magic, 4); + uint32_t n = size; + memcpy(buf + 4, &n, 4); + memcpy(buf + 8, auth_key.data(), size); + memcpy(buf + 8 + size, &json_magic, 4); + writer.send(buf, sizeof buf); + } + + ReadLock reader(conn_private.get()); + { + const size_t max_response_length = 1024; + char buf[max_response_length + 1]; + size_t len = reader.recv_cstring(buf, max_response_length); + if (len == max_response_length || strcmp(buf, "SUCCESS")) { + buf[len] = 0; + ::close(sockfd); + throw Error("Server rejected connection with message: %s", buf); + } + } + + return std::unique_ptr(new Connection(conn_private.release())); +} + +Connection::Connection(ConnectionPrivate *dd) : d(dd) { } +Connection::~Connection() { + // close(); +} + +size_t ReadLock::recv_some(char* buf, size_t size, double wait) { + if (wait != FOREVER) { + while (true) { + fd_set readfds; + struct timeval tv; + + FD_ZERO(&readfds); + FD_SET(conn->guarded_sockfd, &readfds); + + tv.tv_sec = (int)wait; + tv.tv_usec = (int)((wait - (int)wait) / MICROSECOND); + int rv = select(conn->guarded_sockfd + 1, &readfds, NULL, NULL, &tv); + if (rv == -1) { + throw Error::from_errno("select"); + } else if (rv == 0) { + throw TimeoutException(); + } + + if (FD_ISSET(conn->guarded_sockfd, &readfds)) { + break; + } + } + } + + ssize_t numbytes = ::recv(conn->guarded_sockfd, buf, size, 0); + if (numbytes == -1) throw Error::from_errno("recv"); + if (debug_net > 1) { + fprintf(stderr, "<< %s\n", write_datum(std::string(buf, numbytes)).c_str()); + } + + return numbytes; +} + +void ReadLock::recv(char* buf, size_t size, double wait) { + while (size) { + size_t numbytes = recv_some(buf, size, wait); + + buf += numbytes; + size -= numbytes; + } +} + +size_t ReadLock::recv_cstring(char* buf, size_t max_size){ + size_t size = 0; + for (; size < max_size; size++) { + recv(buf, 1, FOREVER); + if (*buf == 0) { + break; + } + buf++; + } + return size; +} + +void WriteLock::send(const char* buf, size_t size) { + while (size) { + ssize_t numbytes = ::write(conn->guarded_sockfd, buf, size); + if (numbytes == -1) throw Error::from_errno("write"); + if (debug_net > 1) { + fprintf(stderr, ">> %s\n", write_datum(std::string(buf, numbytes)).c_str()); + } + + buf += numbytes; + size -= numbytes; + } +} + +void WriteLock::send(const std::string data) { + send(data.data(), data.size()); +} + +std::string ReadLock::recv(size_t size) { + char buf[size]; + recv(buf, size, FOREVER); + return buf; +} + +void Connection::close() { + CacheLock guard(d.get()); + for (auto& it : d->guarded_cache) { + stop_query(it.first); + } + + int ret = ::close(d->guarded_sockfd); + if (ret == -1) { + throw Error::from_errno("close"); + } +} + +Response ConnectionPrivate::wait_for_response(uint64_t token_want, double wait) { + CacheLock guard(this); + ConnectionPrivate::TokenCache& cache = guarded_cache[token_want]; + + while (true) { + if (!cache.responses.empty()) { + Response response(std::move(cache.responses.front())); + cache.responses.pop(); + if (cache.closed && cache.responses.empty()) { + guarded_cache.erase(token_want); + } + + return response; + } + + if (cache.closed) { + throw Error("Trying to read from a closed token"); + } + + if (guarded_loop_active) { + cache.cond.wait(guard.inner_lock); + } else { + break; + } + } + + ReadLock reader(this); + return reader.read_loop(token_want, std::move(guard), wait); +} + +Response ReadLock::read_loop(uint64_t token_want, CacheLock&& guard, double wait) { + if (!guard.inner_lock) { + guard.lock(); + } + if (conn->guarded_loop_active) { + throw Error("Cannot run more than one read loop on the same connection"); + } + conn->guarded_loop_active = true; + guard.unlock(); + + try { + while (true) { + char buf[12]; + bzero(buf, sizeof(buf)); + recv(buf, 12, wait); + uint64_t token_got; + memcpy(&token_got, buf, 8); + uint32_t length; + memcpy(&length, buf + 8, 4); + + std::unique_ptr bufmem(new char[length + 1]); + char *buffer = bufmem.get(); + bzero(buffer, length + 1); + recv(buffer, length, wait); + buffer[length] = '\0'; + + rapidjson::Document json; + json.ParseInsitu(buffer); + if (json.HasParseError()) { + fprintf(stderr, "json parse error, code: %d, position: %d\n", + (int)json.GetParseError(), (int)json.GetErrorOffset()); + } else if (json.IsNull()) { + fprintf(stderr, "null value, read: %s\n", buffer); + } + + Datum datum = read_datum(json); + if (debug_net > 0) { + fprintf(stderr, "[%" PRIu64 "] << %s\n", token_got, write_datum(datum).c_str()); + } + + Response response(std::move(datum)); + + if (token_got == token_want) { + guard.lock(); + if (response.type != Protocol::Response::ResponseType::SUCCESS_PARTIAL) { + auto it = conn->guarded_cache.find(token_got); + if (it != conn->guarded_cache.end()) { + it->second.closed = true; + it->second.cond.notify_all(); + } + conn->guarded_cache.erase(it); + } + conn->guarded_loop_active = false; + for (auto& it : conn->guarded_cache) { + it.second.cond.notify_all(); + } + return response; + } else { + guard.lock(); + auto it = conn->guarded_cache.find(token_got); + if (it == conn->guarded_cache.end()) { + // drop the response + } else if (!it->second.closed) { + it->second.responses.emplace(std::move(response)); + if (response.type != Protocol::Response::ResponseType::SUCCESS_PARTIAL) { + it->second.closed = true; + } + } + it->second.cond.notify_all(); + guard.unlock(); + } + } + } catch (const TimeoutException &e) { + if (!guard.inner_lock){ + guard.lock(); + } + conn->guarded_loop_active = false; + throw e; + } +} + +void ConnectionPrivate::run_query(Query query, bool no_reply) { + WriteLock writer(this); + writer.send(query.serialize()); +} + +Cursor Connection::start_query(Term *term, OptArgs&& opts) { + bool no_reply = false; + auto it = opts.find("noreply"); + if (it != opts.end()) { + no_reply = *(it->second.datum.get_boolean()); + } + + uint64_t token = d->new_token(); + { + CacheLock guard(d.get()); + d->guarded_cache[token]; + } + + d->run_query(Query{QueryType::START, token, term->datum, std::move(opts)}); + if (no_reply) { + return Cursor(new CursorPrivate(token, this, Nil())); + } + + Cursor cursor(new CursorPrivate(token, this)); + Response response = d->wait_for_response(token, FOREVER); + cursor.d->add_response(std::move(response)); + return cursor; +} + +void Connection::stop_query(uint64_t token) { + const auto& it = d->guarded_cache.find(token); + if (it != d->guarded_cache.end() && !it->second.closed) { + d->run_query(Query{QueryType::STOP, token}, true); + } +} + +void Connection::continue_query(uint64_t token) { + d->run_query(Query{QueryType::CONTINUE, token}, true); +} + +Error Response::as_error() { + std::string repr; + if (result.size() == 1) { + std::string* string = result[0].get_string(); + if (string) { + repr = *string; + } else { + repr = write_datum(result[0]); + } + } else { + repr = write_datum(Datum(result)); + } + std::string err; + using RT = Protocol::Response::ResponseType; + using ET = Protocol::Response::ErrorType; + switch (type) { + case RT::SUCCESS_SEQUENCE: err = "unexpected response: SUCCESS_SEQUENCE"; break; + case RT::SUCCESS_PARTIAL: err = "unexpected response: SUCCESS_PARTIAL"; break; + case RT::SUCCESS_ATOM: err = "unexpected response: SUCCESS_ATOM"; break; + case RT::WAIT_COMPLETE: err = "unexpected response: WAIT_COMPLETE"; break; + case RT::SERVER_INFO: err = "unexpected response: SERVER_INFO"; break; + case RT::CLIENT_ERROR: err = "ReqlDriverError"; break; + case RT::COMPILE_ERROR: err = "ReqlCompileError"; break; + case RT::RUNTIME_ERROR: + switch (error_type) { + case ET::INTERNAL: err = "ReqlInternalError"; break; + case ET::RESOURCE_LIMIT: err = "ReqlResourceLimitError"; break; + case ET::QUERY_LOGIC: err = "ReqlQueryLogicError"; break; + case ET::NON_EXISTENCE: err = "ReqlNonExistenceError"; break; + case ET::OP_FAILED: err = "ReqlOpFailedError"; break; + case ET::OP_INDETERMINATE: err = "ReqlOpIndeterminateError"; break; + case ET::USER: err = "ReqlUserError"; break; + case ET::PERMISSION_ERROR: err = "ReqlPermissionError"; break; + default: err = "ReqlRuntimeError"; break; + } + } + throw Error("%s: %s", err.c_str(), repr.c_str()); +} + +Protocol::Response::ResponseType response_type(double t) { + int n = static_cast(t); + using RT = Protocol::Response::ResponseType; + switch (n) { + case static_cast(RT::SUCCESS_ATOM): + return RT::SUCCESS_ATOM; + case static_cast(RT::SUCCESS_SEQUENCE): + return RT::SUCCESS_SEQUENCE; + case static_cast(RT::SUCCESS_PARTIAL): + return RT::SUCCESS_PARTIAL; + case static_cast(RT::WAIT_COMPLETE): + return RT::WAIT_COMPLETE; + case static_cast(RT::CLIENT_ERROR): + return RT::CLIENT_ERROR; + case static_cast(RT::COMPILE_ERROR): + return RT::COMPILE_ERROR; + case static_cast(RT::RUNTIME_ERROR): + return RT::RUNTIME_ERROR; + default: + throw Error("Unknown response type"); + } +} + +Protocol::Response::ErrorType runtime_error_type(double t) { + int n = static_cast(t); + using ET = Protocol::Response::ErrorType; + switch (n) { + case static_cast(ET::INTERNAL): + return ET::INTERNAL; + case static_cast(ET::RESOURCE_LIMIT): + return ET::RESOURCE_LIMIT; + case static_cast(ET::QUERY_LOGIC): + return ET::QUERY_LOGIC; + case static_cast(ET::NON_EXISTENCE): + return ET::NON_EXISTENCE; + case static_cast(ET::OP_FAILED): + return ET::OP_FAILED; + case static_cast(ET::OP_INDETERMINATE): + return ET::OP_INDETERMINATE; + case static_cast(ET::USER): + return ET::USER; + default: + throw Error("Unknown error type"); + } +} + +} diff --git a/ext/librethinkdbxx/src/connection.h b/ext/librethinkdbxx/src/connection.h new file mode 100644 index 00000000..d3882857 --- /dev/null +++ b/ext/librethinkdbxx/src/connection.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "protocol_defs.h" +#include "datum.h" +#include "error.h" + +#define FOREVER (-1) +#define SECOND 1 +#define MICROSECOND 0.000001 + +namespace RethinkDB { + +class Term; +using OptArgs = std::map; + +// A connection to a RethinkDB server +// It contains: +// * A socket +// * Read and write locks +// * A cache of responses that have not been read by the corresponding Cursor +class ConnectionPrivate; +class Connection { +public: + Connection() = delete; + Connection(const Connection&) noexcept = delete; + Connection(Connection&&) noexcept = delete; + Connection& operator=(Connection&&) noexcept = delete; + Connection& operator=(const Connection&) noexcept = delete; + ~Connection(); + + void close(); + +private: + explicit Connection(ConnectionPrivate *dd); + std::unique_ptr d; + + Cursor start_query(Term *term, OptArgs&& args); + void stop_query(uint64_t); + void continue_query(uint64_t); + + friend class Cursor; + friend class CursorPrivate; + friend class Token; + friend class Term; + friend std::unique_ptr + connect(std::string host, int port, std::string auth_key); + +}; + +// $doc(connect) +std::unique_ptr connect(std::string host = "localhost", int port = 28015, std::string auth_key = ""); + +} diff --git a/ext/librethinkdbxx/src/connection_p.h b/ext/librethinkdbxx/src/connection_p.h new file mode 100644 index 00000000..d8a95e3c --- /dev/null +++ b/ext/librethinkdbxx/src/connection_p.h @@ -0,0 +1,133 @@ +#ifndef CONNECTION_P_H +#define CONNECTION_P_H + +#include + +#include "connection.h" +#include "term.h" +#include "json_p.h" + +namespace RethinkDB { + +extern const int debug_net; + +struct Query { + Protocol::Query::QueryType type; + uint64_t token; + Datum term; + OptArgs optArgs; + + std::string serialize() { + Array query_arr{static_cast(type)}; + if (term.is_valid()) query_arr.emplace_back(term); + if (!optArgs.empty()) + query_arr.emplace_back(Term(std::move(optArgs)).datum); + + std::string query_str = write_datum(query_arr); + if (debug_net > 0) { + fprintf(stderr, "[%" PRIu64 "] >> %s\n", token, query_str.c_str()); + } + + char header[12]; + memcpy(header, &token, 8); + uint32_t size = query_str.size(); + memcpy(header + 8, &size, 4); + query_str.insert(0, header, 12); + return query_str; + } +}; + +// Used internally to convert a raw response type into an enum +Protocol::Response::ResponseType response_type(double t); +Protocol::Response::ErrorType runtime_error_type(double t); + +// Contains a response from the server. Use the Cursor class to interact with these responses +class Response { +public: + Response() = delete; + explicit Response(Datum&& datum) : + type(response_type(std::move(datum).extract_field("t").extract_number())), + error_type(datum.get_field("e") ? + runtime_error_type(std::move(datum).extract_field("e").extract_number()) : + Protocol::Response::ErrorType(0)), + result(std::move(datum).extract_field("r").extract_array()) { } + Error as_error(); + Protocol::Response::ResponseType type; + Protocol::Response::ErrorType error_type; + Array result; +}; + +class Token; +class ConnectionPrivate { +public: + ConnectionPrivate(int sockfd) + : guarded_next_token(1), guarded_sockfd(sockfd), guarded_loop_active(false) + { } + + void run_query(Query query, bool no_reply = false); + + Response wait_for_response(uint64_t, double); + uint64_t new_token() { + return guarded_next_token++; + } + + std::mutex read_lock; + std::mutex write_lock; + std::mutex cache_lock; + + struct TokenCache { + bool closed = false; + std::condition_variable cond; + std::queue responses; + }; + + std::map guarded_cache; + uint64_t guarded_next_token; + int guarded_sockfd; + bool guarded_loop_active; +}; + +class CacheLock { +public: + CacheLock(ConnectionPrivate* conn) : inner_lock(conn->cache_lock) { } + + void lock() { + inner_lock.lock(); + } + + void unlock() { + inner_lock.unlock(); + } + + std::unique_lock inner_lock; +}; + +class ReadLock { +public: + ReadLock(ConnectionPrivate* conn_) : lock(conn_->read_lock), conn(conn_) { } + + size_t recv_some(char*, size_t, double wait); + void recv(char*, size_t, double wait); + std::string recv(size_t); + size_t recv_cstring(char*, size_t); + + Response read_loop(uint64_t, CacheLock&&, double); + + std::lock_guard lock; + ConnectionPrivate* conn; +}; + +class WriteLock { +public: + WriteLock(ConnectionPrivate* conn_) : lock(conn_->write_lock), conn(conn_) { } + + void send(const char*, size_t); + void send(std::string); + + std::lock_guard lock; + ConnectionPrivate* conn; +}; + +} // namespace RethinkDB + +#endif // CONNECTION_P_H diff --git a/ext/librethinkdbxx/src/cursor.cc b/ext/librethinkdbxx/src/cursor.cc new file mode 100644 index 00000000..987c4dba --- /dev/null +++ b/ext/librethinkdbxx/src/cursor.cc @@ -0,0 +1,221 @@ +#include "cursor.h" +#include "cursor_p.h" +#include "exceptions.h" + +namespace RethinkDB { + +// for type completion, in order to forward declare with unique_ptr +Cursor::Cursor(Cursor&&) = default; +Cursor& Cursor::operator=(Cursor&&) = default; + +CursorPrivate::CursorPrivate(uint64_t token_, Connection *conn_) + : single(false), no_more(false), index(0), + token(token_), conn(conn_) +{ } + +CursorPrivate::CursorPrivate(uint64_t token_, Connection *conn_, Datum&& datum) + : single(true), no_more(true), index(0), buffer(Array{std::move(datum)}), + token(token_), conn(conn_) +{ } + +Cursor::Cursor(CursorPrivate *dd) : d(dd) {} + +Cursor::~Cursor() { + if (d && d->conn) { + close(); + } +} + +Datum& Cursor::next(double wait) const { + if (!has_next(wait)) { + throw Error("next: No more data"); + } + + return d->buffer[d->index++]; +} + +Datum& Cursor::peek(double wait) const { + if (!has_next(wait)) { + throw Error("next: No more data"); + } + + return d->buffer[d->index]; +} + +void Cursor::each(std::function f, double wait) const { + while (has_next(wait)) { + f(std::move(d->buffer[d->index++])); + } +} + +void CursorPrivate::convert_single() const { + if (index != 0) { + throw Error("Cursor: already consumed"); + } + + if (buffer.size() != 1) { + throw Error("Cursor: invalid response from server"); + } + + if (!buffer[0].is_array()) { + throw Error("Cursor: not an array"); + } + + buffer.swap(buffer[0].extract_array()); + single = false; +} + +void CursorPrivate::clear_and_read_all() const { + if (single) { + convert_single(); + } + if (index != 0) { + buffer.erase(buffer.begin(), buffer.begin() + index); + index = 0; + } + while (!no_more) { + add_response(conn->d->wait_for_response(token, FOREVER)); + } +} + +Array&& Cursor::to_array() && { + d->clear_and_read_all(); + return std::move(d->buffer); +} + +Array Cursor::to_array() const & { + d->clear_and_read_all(); + return d->buffer; +} + +Datum Cursor::to_datum() const & { + if (d->single) { + if (d->index != 0) { + throw Error("to_datum: already consumed"); + } + return d->buffer[0]; + } + + d->clear_and_read_all(); + return d->buffer; +} + +Datum Cursor::to_datum() && { + Datum ret((Nil())); + if (d->single) { + if (d->index != 0) { + throw Error("to_datum: already consumed"); + } + ret = std::move(d->buffer[0]); + } else { + d->clear_and_read_all(); + ret = std::move(d->buffer); + } + + return ret; +} + +void Cursor::close() const { + d->conn->stop_query(d->token); + d->no_more = true; +} + +bool Cursor::has_next(double wait) const { + if (d->single) { + d->convert_single(); + } + + while (true) { + if (d->index >= d->buffer.size()) { + if (d->no_more) { + return false; + } + d->add_response(d->conn->d->wait_for_response(d->token, wait)); + } else { + return true; + } + } +} + +bool Cursor::is_single() const { + return d->single; +} + +void CursorPrivate::add_results(Array&& results) const { + if (index >= buffer.size()) { + buffer = std::move(results); + index = 0; + } else { + for (auto& it : results) { + buffer.emplace_back(std::move(it)); + } + } +} + +void CursorPrivate::add_response(Response&& response) const { + using RT = Protocol::Response::ResponseType; + switch (response.type) { + case RT::SUCCESS_SEQUENCE: + add_results(std::move(response.result)); + no_more = true; + break; + case RT::SUCCESS_PARTIAL: + conn->continue_query(token); + add_results(std::move(response.result)); + break; + case RT::SUCCESS_ATOM: + add_results(std::move(response.result)); + single = true; + no_more = true; + break; + case RT::SERVER_INFO: + add_results(std::move(response.result)); + single = true; + no_more = true; + break; + case RT::WAIT_COMPLETE: + case RT::CLIENT_ERROR: + case RT::COMPILE_ERROR: + case RT::RUNTIME_ERROR: + no_more = true; + throw response.as_error(); + } +} + +Cursor::iterator Cursor::begin() { + return iterator(this); +} + +Cursor::iterator Cursor::end() { + return iterator(nullptr); +} + +Cursor::iterator::iterator(Cursor* cursor_) : cursor(cursor_) {} + +Cursor::iterator& Cursor::iterator::operator++ () { + if (cursor == nullptr) { + throw Error("incrementing an exhausted Cursor iterator"); + } + + cursor->next(); + return *this; +} + +Datum& Cursor::iterator::operator* () { + if (cursor == nullptr) { + throw Error("reading from empty Cursor iterator"); + } + + return cursor->peek(); +} + +bool Cursor::iterator::operator!= (const Cursor::iterator& other) const { + if (cursor == other.cursor) { + return false; + } + + return !((cursor == nullptr && !other.cursor->has_next()) || + (other.cursor == nullptr && !cursor->has_next())); +} + +} diff --git a/ext/librethinkdbxx/src/cursor.h b/ext/librethinkdbxx/src/cursor.h new file mode 100644 index 00000000..60ae1817 --- /dev/null +++ b/ext/librethinkdbxx/src/cursor.h @@ -0,0 +1,76 @@ +#pragma once + +#include "connection.h" + +namespace RethinkDB { + +// The response from the server, as returned by run. +// The response is either a single datum or a stream: +// * If it is a stream, the cursor represents each element of the stream. +// - Batches are fetched from the server as needed. +// * If it is a single datum, is_single() returns true. +// - If it is an array, the cursor represents each element of that array +// - Otherwise, to_datum() returns the datum and iteration throws an exception. +// The cursor can only be iterated over once, it discards data that has already been read. +class CursorPrivate; +class Cursor { +public: + Cursor() = delete; + ~Cursor(); + + Cursor(Cursor&&); // movable + Cursor& operator=(Cursor&&); + Cursor(const Cursor&) = delete; // not copyable + Cursor& operator=(const Cursor&) = delete; + + // Returned by begin() and end() + class iterator { + public: + iterator(Cursor*); + iterator& operator++ (); + Datum& operator* (); + bool operator!= (const iterator&) const; + + private: + Cursor *cursor; + }; + + // Consume the next element + Datum& next(double wait = FOREVER) const; + + // Peek at the next element + Datum& peek(double wait = FOREVER) const; + + // Call f on every element of the Cursor + void each(std::function f, double wait = FOREVER) const; + + // Consume and return all elements + Array&& to_array() &&; + + // If is_single(), returns the single datum. Otherwise returns to_array(). + Datum to_datum() &&; + Datum to_datum() const &; + + // Efficiently consume and return all elements + Array to_array() const &; + + // Close the cursor + void close() const; + + // Returns false if there are no more elements + bool has_next(double wait = FOREVER) const; + + // Returns false if the cursor is a stream + bool is_single() const; + + iterator begin(); + iterator end(); + +private: + explicit Cursor(CursorPrivate *dd); + std::unique_ptr d; + + friend class Connection; +}; + +} diff --git a/ext/librethinkdbxx/src/cursor_p.h b/ext/librethinkdbxx/src/cursor_p.h new file mode 100644 index 00000000..ce584cd7 --- /dev/null +++ b/ext/librethinkdbxx/src/cursor_p.h @@ -0,0 +1,29 @@ +#ifndef CURSOR_P_H +#define CURSOR_P_H + +#include "connection_p.h" + +namespace RethinkDB { + +class CursorPrivate { +public: + CursorPrivate(uint64_t token, Connection *conn); + CursorPrivate(uint64_t token, Connection *conn, Datum&&); + + void add_response(Response&&) const; + void add_results(Array&&) const; + void clear_and_read_all() const; + void convert_single() const; + + mutable bool single = false; + mutable bool no_more = false; + mutable size_t index = 0; + mutable Array buffer; + + uint64_t token; + Connection *conn; +}; + +} // namespace RethinkDB + +#endif // CURSOR_P_H \ No newline at end of file diff --git a/ext/librethinkdbxx/src/datum.cc b/ext/librethinkdbxx/src/datum.cc new file mode 100644 index 00000000..e4dbc8dc --- /dev/null +++ b/ext/librethinkdbxx/src/datum.cc @@ -0,0 +1,449 @@ +#include +#include + +#include "datum.h" +#include "json_p.h" +#include "utils.h" +#include "cursor.h" + +#include "rapidjson-config.h" +#include "rapidjson/prettywriter.h" +#include "rapidjson/stringbuffer.h" + +namespace RethinkDB { + +using TT = Protocol::Term::TermType; + +bool Datum::is_nil() const { + return type == Type::NIL; +} + +bool Datum::is_boolean() const { + return type == Type::BOOLEAN; +} + +bool Datum::is_number() const { + return type == Type::NUMBER; +} + +bool Datum::is_string() const { + return type == Type::STRING; +} + +bool Datum::is_object() const { + return type == Type::OBJECT; +} + +bool Datum::is_array() const { + return type == Type::ARRAY; +} + +bool Datum::is_binary() const { + return type == Type::BINARY; +} + +bool Datum::is_time() const { + return type == Type::TIME; +} + +bool* Datum::get_boolean() { + if (type == Type::BOOLEAN) { + return &value.boolean; + } else { + return NULL; + } +} + +const bool* Datum::get_boolean() const { + if (type == Type::BOOLEAN) { + return &value.boolean; + } else { + return NULL; + } +} + +double* Datum::get_number() { + if (type == Type::NUMBER) { + return &value.number; + } else { + return NULL; + } +} + +const double* Datum::get_number() const { + if (type == Type::NUMBER) { + return &value.number; + } else { + return NULL; + } +} + +std::string* Datum::get_string() { + if (type == Type::STRING) { + return &value.string; + } else { + return NULL; + } +} + +const std::string* Datum::get_string() const { + if (type == Type::STRING) { + return &value.string; + } else { + return NULL; + } +} + +Datum* Datum::get_field(std::string key) { + if (type != Type::OBJECT) { + return NULL; + } + auto it = value.object.find(key); + if (it == value.object.end()) { + return NULL; + } + return &it->second; +} + +const Datum* Datum::get_field(std::string key) const { + if (type != Type::OBJECT) { + return NULL; + } + auto it = value.object.find(key); + if (it == value.object.end()) { + return NULL; + } + return &it->second; +} + +Datum* Datum::get_nth(size_t i) { + if (type != Type::ARRAY) { + return NULL; + } + if (i >= value.array.size()) { + return NULL; + } + return &value.array[i]; +} + +const Datum* Datum::get_nth(size_t i) const { + if (type != Type::ARRAY) { + return NULL; + } + if (i >= value.array.size()) { + return NULL; + } + return &value.array[i]; +} + +Object* Datum::get_object() { + if (type == Type::OBJECT) { + return &value.object; + } else { + return NULL; + } +} + +const Object* Datum::get_object() const { + if (type == Type::OBJECT) { + return &value.object; + } else { + return NULL; + } +} + +Array* Datum::get_array() { + if (type == Type::ARRAY) { + return &value.array; + } else { + return NULL; + } +} + +const Array* Datum::get_array() const { + if (type == Type::ARRAY) { + return &value.array; + } else { + return NULL; + } +} + +Binary* Datum::get_binary() { + if (type == Type::BINARY) { + return &value.binary; + } else { + return NULL; + } +} + +const Binary* Datum::get_binary() const { + if (type == Type::BINARY) { + return &value.binary; + } else { + return NULL; + } +} + +Time* Datum::get_time() { + if (type == Type::TIME) { + return &value.time; + } else { + return NULL; + } +} + +const Time* Datum::get_time() const { + if (type == Type::TIME) { + return &value.time; + } else { + return NULL; + } +} + +bool& Datum::extract_boolean() { + if (type != Type::BOOLEAN) { + throw Error("extract_bool: Not a boolean"); + } + return value.boolean; +} + +double& Datum::extract_number() { + if (type != Type::NUMBER) { + throw Error("extract_number: Not a number: %s", write_datum(*this).c_str()); + } + return value.number; +} + +std::string& Datum::extract_string() { + if (type != Type::STRING) { + throw Error("extract_string: Not a string"); + } + return value.string; +} + +Object& Datum::extract_object() { + if (type != Type::OBJECT) { + throw Error("extract_object: Not an object"); + } + return value.object; +} + +Datum& Datum::extract_field(std::string key) { + if (type != Type::OBJECT) { + throw Error("extract_field: Not an object"); + } + auto it = value.object.find(key); + if (it == value.object.end()) { + throw Error("extract_field: No such key in object"); + } + return it->second; +} + +Datum& Datum::extract_nth(size_t i) { + if (type != Type::ARRAY) { + throw Error("extract_nth: Not an array"); + } + if (i >= value.array.size()) { + throw Error("extract_nth: index too large"); + } + return value.array[i]; +} + +Array& Datum::extract_array() { + if (type != Type::ARRAY) { + throw Error("get_array: Not an array"); + } + return value.array; +} + +Binary& Datum::extract_binary() { + if (type != Type::BINARY) { + throw Error("get_binary: Not a binary"); + } + return value.binary; +} + +Time& Datum::extract_time() { + if (type != Type::TIME) { + throw Error("get_time: Not a time"); + } + return value.time; +} + +int Datum::compare(const Datum& other) const { +#define COMPARE(a, b) do { \ + if (a < b) { return -1; } \ + if (a > b) { return 1; } } while(0) +#define COMPARE_OTHER(x) COMPARE(x, other.x) + + COMPARE_OTHER(type); + int c; + switch (type) { + case Type::NIL: case Type::INVALID: break; + case Type::BOOLEAN: COMPARE_OTHER(value.boolean); break; + case Type::NUMBER: COMPARE_OTHER(value.number); break; + case Type::STRING: + c = value.string.compare(other.value.string); + COMPARE(c, 0); + break; + case Type::BINARY: + c = value.binary.data.compare(other.value.binary.data); + COMPARE(c, 0); + break; + case Type::TIME: + COMPARE(value.time.epoch_time, other.value.time.epoch_time); + COMPARE(value.time.utc_offset, other.value.time.utc_offset); + break; + case Type::ARRAY: + COMPARE_OTHER(value.array.size()); + for (size_t i = 0; i < value.array.size(); i++) { + c = value.array[i].compare(other.value.array[i]); + COMPARE(c, 0); + } + break; + case Type::OBJECT: + COMPARE_OTHER(value.object.size()); + for (Object::const_iterator l = value.object.begin(), + r = other.value.object.begin(); + l != value.object.end(); + ++l, ++r) { + COMPARE(l->first, r->first); + c = l->second.compare(r->second); + COMPARE(c, 0); + } + break; + default: + throw Error("cannot compare invalid datum"); + } + return 0; +#undef COMPARE_OTHER +#undef COMPARE +} + +bool Datum::operator== (const Datum& other) const { + return compare(other) == 0; +} + +Datum Datum::from_raw() const { + do { + const Datum* type_field = get_field("$reql_type$"); + if (!type_field) break; + const std::string* type = type_field->get_string(); + if (!type) break;; + if (!strcmp(type->c_str(), "BINARY")) { + const Datum* data_field = get_field("data"); + if (!data_field) break; + const std::string* encoded_data = data_field->get_string(); + if (!encoded_data) break; + Binary binary(""); + if (base64_decode(*encoded_data, binary.data)) { + return binary; + } + } else if (!strcmp(type->c_str(), "TIME")) { + const Datum* epoch_field = get_field("epoch_time"); + if (!epoch_field) break; + const Datum* tz_field = get_field("timezone"); + if (!tz_field) break; + const double* epoch_time = epoch_field->get_number(); + if (!epoch_time) break; + const std::string* tz = tz_field->get_string(); + if (!tz) break; + double offset; + if (!Time::parse_utc_offset(*tz, &offset)) break; + return Time(*epoch_time, offset); + } + } while (0); + return *this; +} + +Datum Datum::to_raw() const { + if (type == Type::BINARY) { + return Object{ + {"$reql_type$", "BINARY"}, + {"data", base64_encode(value.binary.data)}}; + } else if (type == Type::TIME) { + return Object{ + {"$reql_type$", "TIME"}, + {"epoch_time", value.time.epoch_time}, + {"timezone", Time::utc_offset_string(value.time.utc_offset)}}; + } + return *this; +} + +Datum::Datum(Cursor&& cursor) : Datum(cursor.to_datum()) { } +Datum::Datum(const Cursor& cursor) : Datum(cursor.to_datum()) { } + +static const double max_dbl_int = 0x1LL << DBL_MANT_DIG; +static const double min_dbl_int = max_dbl_int * -1; +bool number_as_integer(double d, int64_t *i_out) { + static_assert(DBL_MANT_DIG == 53, "Doubles are wrong size."); + + if (min_dbl_int <= d && d <= max_dbl_int) { + int64_t i = d; + if (static_cast(i) == d) { + *i_out = i; + return true; + } + } + return false; +} + +template void Datum::write_json( + rapidjson::Writer *writer) const; +template void Datum::write_json( + rapidjson::PrettyWriter *writer) const; + +template +void Datum::write_json(json_writer_t *writer) const { + switch (type) { + case Type::NIL: writer->Null(); break; + case Type::BOOLEAN: writer->Bool(value.boolean); break; + case Type::NUMBER: { + const double d = value.number; + // Always print -0.0 as a double since integers cannot represent -0. + // Otherwise check if the number is an integer and print it as such. + int64_t i; + if (!(d == 0.0 && std::signbit(d)) && number_as_integer(d, &i)) { + writer->Int64(i); + } else { + writer->Double(d); + } + } break; + case Type::STRING: writer->String(value.string.data(), value.string.size()); break; + case Type::ARRAY: { + writer->StartArray(); + for (auto it : value.array) { + it.write_json(writer); + } + writer->EndArray(); + } break; + case Type::OBJECT: { + writer->StartObject(); + for (auto it : value.object) { + writer->Key(it.first.data(), it.first.size()); + it.second.write_json(writer); + } + writer->EndObject(); + } break; + + case Type::BINARY: + case Type::TIME: + to_raw().write_json(writer); + break; + default: + throw Error("cannot write invalid datum"); + } +} + +std::string Datum::as_json() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + write_json(&writer); + return std::string(buffer.GetString(), buffer.GetSize()); +} + +Datum Datum::from_json(const std::string& json) { + return read_datum(json); +} + +} // namespace RethinkDB diff --git a/ext/librethinkdbxx/src/datum.h b/ext/librethinkdbxx/src/datum.h new file mode 100644 index 00000000..051e2ca2 --- /dev/null +++ b/ext/librethinkdbxx/src/datum.h @@ -0,0 +1,287 @@ +#pragma once + +#include +#include +#include +#include + +#include "protocol_defs.h" +#include "error.h" +#include "types.h" + +namespace RethinkDB { + +class Cursor; + +// The type of data stored in a RethinkDB database. +// The following JSON types are represented in a Datum as +// * null -> Nil +// * boolean -> bool +// * number -> double +// * unicode strings -> std::string +// * array -> Array (aka std::vector +// * object -> Object (aka std::map> +// Datums can also contain one of the following extra types +// * binary strings -> Binary +// * timestamps -> Time +// * points. lines and polygons -> not implemented +class Datum { +public: + Datum() : type(Type::INVALID), value() {} + Datum(Nil) : type(Type::NIL), value() { } + Datum(bool boolean_) : type(Type::BOOLEAN), value(boolean_) { } + Datum(double number_) : type(Type::NUMBER), value(number_) { } + Datum(const std::string& string_) : type(Type::STRING), value(string_) { } + Datum(std::string&& string_) : type(Type::STRING), value(std::move(string_)) { } + Datum(const Array& array_) : type(Type::ARRAY), value(array_) { } + Datum(Array&& array_) : type(Type::ARRAY), value(std::move(array_)) { } + Datum(const Binary& binary) : type(Type::BINARY), value(binary) { } + Datum(Binary&& binary) : type(Type::BINARY), value(std::move(binary)) { } + Datum(const Time time) : type(Type::TIME), value(time) { } + Datum(const Object& object_) : type(Type::OBJECT), value(object_) { } + Datum(Object&& object_) : type(Type::OBJECT), value(std::move(object_)) { } + Datum(const Datum& other) : type(other.type), value(other.type, other.value) { } + Datum(Datum&& other) : type(other.type), value(other.type, std::move(other.value)) { } + + Datum& operator=(const Datum& other) { + value.destroy(type); + type = other.type; + value.set(type, other.value); + return *this; + } + + Datum& operator=(Datum&& other) { + value.destroy(type); + type = other.type; + value.set(type, std::move(other.value)); + return *this; + } + + Datum(unsigned short number_) : Datum(static_cast(number_)) { } + Datum(signed short number_) : Datum(static_cast(number_)) { } + Datum(unsigned int number_) : Datum(static_cast(number_)) { } + Datum(signed int number_) : Datum(static_cast(number_)) { } + Datum(unsigned long number_) : Datum(static_cast(number_)) { } + Datum(signed long number_) : Datum(static_cast(number_)) { } + Datum(unsigned long long number_) : Datum(static_cast(number_)) { } + Datum(signed long long number_) : Datum(static_cast(number_)) { } + + Datum(Protocol::Term::TermType type) : Datum(static_cast(type)) { } + Datum(const char* string) : Datum(static_cast(string)) { } + + // Cursors are implicitly converted into datums + Datum(Cursor&&); + Datum(const Cursor&); + + template + Datum(const std::map& map) : type(Type::OBJECT), value(Object()) { + for (const auto& it : map) { + value.object.emplace(it.left, Datum(it.right)); + } + } + + template + Datum(std::map&& map) : type(Type::OBJECT), value(Object()) { + for (auto& it : map) { + value.object.emplace(it.first, Datum(std::move(it.second))); + } + } + + template + Datum(const std::vector& vec) : type(Type::ARRAY), value(Array()) { + for (const auto& it : vec) { + value.array.emplace_back(it); + } + } + + template + Datum(std::vector&& vec) : type(Type::ARRAY), value(Array()) { + for (auto& it : vec) { + value.array.emplace_back(std::move(it)); + } + } + + ~Datum() { + value.destroy(type); + } + + // Apply a visitor + template + R apply(F f, A&& ...args) const & { + switch (type) { + case Type::NIL: return f(Nil(), std::forward(args)...); break; + case Type::BOOLEAN: return f(value.boolean, std::forward(args)...); break; + case Type::NUMBER: return f(value.number, std::forward(args)...); break; + case Type::STRING: return f(value.string, std::forward(args)...); break; + case Type::OBJECT: return f(value.object, std::forward(args)...); break; + case Type::ARRAY: return f(value.array, std::forward(args)...); break; + case Type::BINARY: return f(value.binary, std::forward(args)...); break; + case Type::TIME: return f(value.time, std::forward(args)...); break; + default: + throw Error("internal error: no such datum type %d", static_cast(type)); + } + } + + template + R apply(F f, A&& ...args) && { + switch (type) { + case Type::NIL: return f(Nil(), std::forward(args)...); break; + case Type::BOOLEAN: return f(std::move(value.boolean), std::forward(args)...); break; + case Type::NUMBER: return f(std::move(value.number), std::forward(args)...); break; + case Type::STRING: return f(std::move(value.string), std::forward(args)...); break; + case Type::OBJECT: return f(std::move(value.object), std::forward(args)...); break; + case Type::ARRAY: return f(std::move(value.array), std::forward(args)...); break; + case Type::BINARY: return f(std::move(value.binary), std::forward(args)...); break; + case Type::TIME: return f(std::move(value.time), std::forward(args)...); break; + default: + throw Error("internal error: no such datum type %d", static_cast(type)); + } + } + + bool is_nil() const; + bool is_boolean() const; + bool is_number() const; + bool is_string() const; + bool is_object() const; + bool is_array() const; + bool is_binary() const; + bool is_time() const; + + // get_* returns nullptr if the datum has a different type + + bool* get_boolean(); + const bool* get_boolean() const; + double* get_number(); + const double* get_number() const; + std::string* get_string(); + const std::string* get_string() const; + Object* get_object(); + const Object* get_object() const; + Datum* get_field(std::string); + const Datum* get_field(std::string) const; + Array* get_array(); + const Array* get_array() const; + Datum* get_nth(size_t); + const Datum* get_nth(size_t) const; + Binary* get_binary(); + const Binary* get_binary() const; + Time* get_time(); + const Time* get_time() const; + + // extract_* throws an exception if the types don't match + + bool& extract_boolean(); + double& extract_number(); + std::string& extract_string(); + Object& extract_object(); + Datum& extract_field(std::string); + Array& extract_array(); + Datum& extract_nth(size_t); + Binary& extract_binary(); + Time& extract_time(); + + // negative, zero or positive if this datum is smaller, identical or larger than the other one, respectively + // This is meant to match the results of RethinkDB's comparison operators + int compare(const Datum&) const; + + // Deep equality + bool operator== (const Datum&) const; + + // Recusively replace non-JSON types into objects that represent them + Datum to_raw() const; + + // Recursively replace objects with a $reql_type$ field into the datum they represent + Datum from_raw() const; + + template void write_json(json_writer_t *writer) const; + + std::string as_json() const; + static Datum from_json(const std::string&); + + bool is_valid() const { return type != Type::INVALID; } + +private: + enum class Type { + INVALID, // default constructed + ARRAY, BOOLEAN, NIL, NUMBER, OBJECT, BINARY, STRING, TIME + // POINT, LINE, POLYGON + }; + Type type; + + union datum_value { + bool boolean; + double number; + std::string string; + Object object; + Array array; + Binary binary; + Time time; + + datum_value() { } + datum_value(bool boolean_) : boolean(boolean_) { } + datum_value(double number_) : number(number_) { } + datum_value(const std::string& string_) : string(string_) { } + datum_value(std::string&& string_) : string(std::move(string_)) { } + datum_value(const Object& object_) : object(object_) { } + datum_value(Object&& object_) : object(std::move(object_)) { } + datum_value(const Array& array_) : array(array_) { } + datum_value(Array&& array_) : array(std::move(array_)) { } + datum_value(const Binary& binary_) : binary(binary_) { } + datum_value(Binary&& binary_) : binary(std::move(binary_)) { } + datum_value(Time time) : time(std::move(time)) { } + + datum_value(Type type, const datum_value& other){ + set(type, other); + } + + datum_value(Type type, datum_value&& other){ + set(type, std::move(other)); + } + + void set(Type type, datum_value&& other) { + switch(type){ + case Type::NIL: case Type::INVALID: break; + case Type::BOOLEAN: new (this) bool(other.boolean); break; + case Type::NUMBER: new (this) double(other.number); break; + case Type::STRING: new (this) std::string(std::move(other.string)); break; + case Type::OBJECT: new (this) Object(std::move(other.object)); break; + case Type::ARRAY: new (this) Array(std::move(other.array)); break; + case Type::BINARY: new (this) Binary(std::move(other.binary)); break; + case Type::TIME: new (this) Time(std::move(other.time)); break; + } + } + + void set(Type type, const datum_value& other) { + switch(type){ + case Type::NIL: case Type::INVALID: break; + case Type::BOOLEAN: new (this) bool(other.boolean); break; + case Type::NUMBER: new (this) double(other.number); break; + case Type::STRING: new (this) std::string(other.string); break; + case Type::OBJECT: new (this) Object(other.object); break; + case Type::ARRAY: new (this) Array(other.array); break; + case Type::BINARY: new (this) Binary(other.binary); break; + case Type::TIME: new (this) Time(other.time); break; + } + } + + void destroy(Type type) { + switch(type){ + case Type::INVALID: break; + case Type::NIL: break; + case Type::BOOLEAN: break; + case Type::NUMBER: break; + case Type::STRING: { typedef std::string str; string.~str(); } break; + case Type::OBJECT: object.~Object(); break; + case Type::ARRAY: array.~Array(); break; + case Type::BINARY: binary.~Binary(); break; + case Type::TIME: time.~Time(); break; + } + } + + ~datum_value() { } + }; + + datum_value value; +}; + +} diff --git a/ext/librethinkdbxx/src/error.h b/ext/librethinkdbxx/src/error.h new file mode 100644 index 00000000..ab75e248 --- /dev/null +++ b/ext/librethinkdbxx/src/error.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + +namespace RethinkDB { + +// All errors thrown by the server have this type +struct Error { + template + explicit Error(const char* format_, T... A) { + format(format_, A...); + } + + Error() = default; + Error(Error&&) = default; + Error(const Error&) = default; + + Error& operator= (Error&& other) { + message = std::move(other.message); + return *this; + } + + static Error from_errno(const char* str){ + return Error("%s: %s", str, strerror(errno)); + } + + // The error message + std::string message; + +private: + const size_t max_message_size = 2048; + + void format(const char* format_, ...) { + va_list args; + va_start(args, format_); + char message_[max_message_size]; + vsnprintf(message_, max_message_size, format_, args); + va_end(args); + message = message_; + } +}; + +} diff --git a/ext/librethinkdbxx/src/exceptions.h b/ext/librethinkdbxx/src/exceptions.h new file mode 100644 index 00000000..08c0b0a0 --- /dev/null +++ b/ext/librethinkdbxx/src/exceptions.h @@ -0,0 +1,13 @@ +#ifndef EXCEPTIONS_H +#define EXCEPTIONS_H + +namespace RethinkDB { + +class TimeoutException : public std::exception { +public: + const char *what() const throw () { return "operation timed out"; } +}; + +} + +#endif // EXCEPTIONS_H diff --git a/ext/librethinkdbxx/src/json.cc b/ext/librethinkdbxx/src/json.cc new file mode 100644 index 00000000..c908eefb --- /dev/null +++ b/ext/librethinkdbxx/src/json.cc @@ -0,0 +1,62 @@ +#include "json_p.h" +#include "error.h" +#include "utils.h" + +#include "rapidjson-config.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +namespace RethinkDB { + +Datum read_datum(const std::string& json) { + rapidjson::Document document; + document.Parse(json); + return read_datum(document); +} + +Datum read_datum(const rapidjson::Value &json) { + switch(json.GetType()) { + case rapidjson::kNullType: return Nil(); + case rapidjson::kFalseType: return false; + case rapidjson::kTrueType: return true; + case rapidjson::kNumberType: return json.GetDouble(); + case rapidjson::kStringType: + return std::string(json.GetString(), json.GetStringLength()); + + case rapidjson::kObjectType: { + Object result; + for (rapidjson::Value::ConstMemberIterator it = json.MemberBegin(); + it != json.MemberEnd(); ++it) { + result.insert(std::make_pair(std::string(it->name.GetString(), + it->name.GetStringLength()), + read_datum(it->value))); + } + + if (result.count("$reql_type$")) + return Datum(std::move(result)).from_raw(); + return std::move(result); + } break; + case rapidjson::kArrayType: { + Array result; + result.reserve(json.Size()); + for (rapidjson::Value::ConstValueIterator it = json.Begin(); + it != json.End(); ++it) { + result.push_back(read_datum(*it)); + } + return std::move(result); + } break; + default: + throw Error("invalid rapidjson value"); + } +} + +std::string write_datum(const Datum& datum) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + datum.write_json(&writer); + return std::string(buffer.GetString(), buffer.GetSize()); +} + +} diff --git a/ext/librethinkdbxx/src/json_p.h b/ext/librethinkdbxx/src/json_p.h new file mode 100644 index 00000000..ebf537a9 --- /dev/null +++ b/ext/librethinkdbxx/src/json_p.h @@ -0,0 +1,19 @@ +#pragma once + +#include "datum.h" + +namespace rapidjson { + class CrtAllocator; + template struct UTF8; + template class GenericValue; + template class MemoryPoolAllocator; + typedef GenericValue, MemoryPoolAllocator > Value; +} + +namespace RethinkDB { + +Datum read_datum(const std::string&); +Datum read_datum(const rapidjson::Value &json); +std::string write_datum(const Datum&); + +} diff --git a/ext/librethinkdbxx/src/rapidjson-config.h b/ext/librethinkdbxx/src/rapidjson-config.h new file mode 100644 index 00000000..320c4048 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson-config.h @@ -0,0 +1,8 @@ +#pragma once + +#define RAPIDJSON_HAS_STDSTRING 1 +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#define RAPIDJSON_HAS_CXX11_TYPETRAITS 1 +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseFullPrecisionFlag diff --git a/ext/librethinkdbxx/src/rapidjson/allocators.h b/ext/librethinkdbxx/src/rapidjson/allocators.h new file mode 100644 index 00000000..c7059697 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/allocators.h @@ -0,0 +1,263 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ALLOCATORS_H_ +#define RAPIDJSON_ALLOCATORS_H_ + +#include "rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Allocator + +/*! \class rapidjson::Allocator + \brief Concept for allocating, resizing and freeing memory block. + + Note that Malloc() and Realloc() are non-static but Free() is static. + + So if an allocator need to support Free(), it needs to put its pointer in + the header of memory block. + +\code +concept Allocator { + static const bool kNeedFree; //!< Whether this allocator needs to call Free(). + + // Allocate a memory block. + // \param size of the memory block in bytes. + // \returns pointer to the memory block. + void* Malloc(size_t size); + + // Resize a memory block. + // \param originalPtr The pointer to current memory block. Null pointer is permitted. + // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) + // \param newSize the new size in bytes. + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); + + // Free a memory block. + // \param pointer to the memory block. Null pointer is permitted. + static void Free(void *ptr); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// CrtAllocator + +//! C-runtime library allocator. +/*! This class is just wrapper for standard C library memory routines. + \note implements Allocator concept +*/ +class CrtAllocator { +public: + static const bool kNeedFree = true; + void* Malloc(size_t size) { + if (size) // behavior of malloc(0) is implementation defined. + return std::malloc(size); + else + return NULL; // standardize to returning NULL. + } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + (void)originalSize; + if (newSize == 0) { + std::free(originalPtr); + return NULL; + } + return std::realloc(originalPtr, newSize); + } + static void Free(void *ptr) { std::free(ptr); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// MemoryPoolAllocator + +//! Default memory allocator used by the parser and DOM. +/*! This allocator allocate memory blocks from pre-allocated memory chunks. + + It does not free memory blocks. And Realloc() only allocate new memory. + + The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. + + User may also supply a buffer as the first chunk. + + If the user-buffer is full then additional chunks are allocated by BaseAllocator. + + The user-buffer is not deallocated by this allocator. + + \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. + \note implements Allocator concept +*/ +template +class MemoryPoolAllocator { +public: + static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + + //! Constructor with chunkSize. + /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + } + + //! Constructor with user-supplied buffer. + /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. + + The user buffer will not be deallocated when this allocator is destructed. + + \param buffer User supplied buffer. + \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). + \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + RAPIDJSON_ASSERT(buffer != 0); + RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); + chunkHead_ = reinterpret_cast(buffer); + chunkHead_->capacity = size - sizeof(ChunkHeader); + chunkHead_->size = 0; + chunkHead_->next = 0; + } + + //! Destructor. + /*! This deallocates all memory chunks, excluding the user-supplied buffer. + */ + ~MemoryPoolAllocator() { + Clear(); + RAPIDJSON_DELETE(ownBaseAllocator_); + } + + //! Deallocates all memory chunks, excluding the user-supplied buffer. + void Clear() { + while (chunkHead_ && chunkHead_ != userBuffer_) { + ChunkHeader* next = chunkHead_->next; + baseAllocator_->Free(chunkHead_); + chunkHead_ = next; + } + if (chunkHead_ && chunkHead_ == userBuffer_) + chunkHead_->size = 0; // Clear user buffer + } + + //! Computes the total capacity of allocated memory chunks. + /*! \return total capacity in bytes. + */ + size_t Capacity() const { + size_t capacity = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + capacity += c->capacity; + return capacity; + } + + //! Computes the memory blocks allocated. + /*! \return total used bytes. + */ + size_t Size() const { + size_t size = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + size += c->size; + return size; + } + + //! Allocates a memory block. (concept Allocator) + void* Malloc(size_t size) { + if (!size) + return NULL; + + size = RAPIDJSON_ALIGN(size); + if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); + + void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; + chunkHead_->size += size; + return buffer; + } + + //! Resizes a memory block (concept Allocator) + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { + if (originalPtr == 0) + return Malloc(newSize); + + if (newSize == 0) + return NULL; + + originalSize = RAPIDJSON_ALIGN(originalSize); + newSize = RAPIDJSON_ALIGN(newSize); + + // Do not shrink if new size is smaller than original + if (originalSize >= newSize) + return originalPtr; + + // Simply expand it if it is the last allocation and there is sufficient space + if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { + size_t increment = static_cast(newSize - originalSize); + if (chunkHead_->size + increment <= chunkHead_->capacity) { + chunkHead_->size += increment; + return originalPtr; + } + } + + // Realloc process: allocate and copy memory, do not free original buffer. + void* newBuffer = Malloc(newSize); + RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. + if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; + } + + //! Frees a memory block (concept Allocator) + static void Free(void *ptr) { (void)ptr; } // Do nothing + +private: + //! Copy constructor is not permitted. + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; + //! Copy assignment operator is not permitted. + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + + //! Creates a new chunk. + /*! \param capacity Capacity of the chunk in bytes. + */ + void AddChunk(size_t capacity) { + if (!baseAllocator_) + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); + ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity)); + chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + } + + static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. + + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + void *userBuffer_; //!< User supplied buffer. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. +}; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/document.h b/ext/librethinkdbxx/src/rapidjson/document.h new file mode 100644 index 00000000..f7f846f2 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/document.h @@ -0,0 +1,2575 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_DOCUMENT_H_ +#define RAPIDJSON_DOCUMENT_H_ + +/*! \file document.h */ + +#include "reader.h" +#include "internal/meta.h" +#include "internal/strfunc.h" +#include "memorystream.h" +#include "encodedstream.h" +#include // placement new +#include + +RAPIDJSON_DIAG_PUSH +#ifdef _MSC_VER +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 6 +RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions +#endif +#endif // __GNUC__ + +#ifndef RAPIDJSON_NOMEMBERITERATORCLASS +#include // std::iterator, std::random_access_iterator_tag +#endif + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +// Forward declaration. +template +class GenericValue; + +template +class GenericDocument; + +//! Name-value pair in a JSON object value. +/*! + This class was internal to GenericValue. It used to be a inner struct. + But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. + https://code.google.com/p/rapidjson/issues/detail?id=64 +*/ +template +struct GenericMember { + GenericValue name; //!< name of member (must be a string) + GenericValue value; //!< value of member. +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericMemberIterator + +#ifndef RAPIDJSON_NOMEMBERITERATORCLASS + +//! (Constant) member iterator for a JSON object value +/*! + \tparam Const Is this a constant iterator? + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. + + This class implements a Random Access Iterator for GenericMember elements + of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. + + \note This iterator implementation is mainly intended to avoid implicit + conversions from iterator values to \c NULL, + e.g. from GenericValue::FindMember. + + \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a + pointer-based implementation, if your platform doesn't provide + the C++ header. + + \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator + */ +template +class GenericMemberIterator + : public std::iterator >::Type> { + + friend class GenericValue; + template friend class GenericMemberIterator; + + typedef GenericMember PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef std::iterator BaseType; + +public: + //! Iterator type itself + typedef GenericMemberIterator Iterator; + //! Constant iterator type + typedef GenericMemberIterator ConstIterator; + //! Non-constant iterator type + typedef GenericMemberIterator NonConstIterator; + + //! Pointer to (const) GenericMember + typedef typename BaseType::pointer Pointer; + //! Reference to (const) GenericMember + typedef typename BaseType::reference Reference; + //! Signed integer type (e.g. \c ptrdiff_t) + typedef typename BaseType::difference_type DifferenceType; + + //! Default constructor (singular value) + /*! Creates an iterator pointing to no element. + \note All operations, except for comparisons, are undefined on such values. + */ + GenericMemberIterator() : ptr_() {} + + //! Iterator conversions to more const + /*! + \param it (Non-const) iterator to copy from + + Allows the creation of an iterator from another GenericMemberIterator + that is "less const". Especially, creating a non-constant iterator + from a constant iterator are disabled: + \li const -> non-const (not ok) + \li const -> const (ok) + \li non-const -> const (ok) + \li non-const -> non-const (ok) + + \note If the \c Const template parameter is already \c false, this + constructor effectively defines a regular copy-constructor. + Otherwise, the copy constructor is implicitly defined. + */ + GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } + + //! @name stepping + //@{ + Iterator& operator++(){ ++ptr_; return *this; } + Iterator& operator--(){ --ptr_; return *this; } + Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } + Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } + //@} + + //! @name increment/decrement + //@{ + Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } + Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } + + Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } + Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } + //@} + + //! @name relations + //@{ + bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } + bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } + bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } + bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } + bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } + bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } + //@} + + //! @name dereference + //@{ + Reference operator*() const { return *ptr_; } + Pointer operator->() const { return ptr_; } + Reference operator[](DifferenceType n) const { return ptr_[n]; } + //@} + + //! Distance + DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } + +private: + //! Internal constructor from plain pointer + explicit GenericMemberIterator(Pointer p) : ptr_(p) {} + + Pointer ptr_; //!< raw pointer +}; + +#else // RAPIDJSON_NOMEMBERITERATORCLASS + +// class-based member iterator implementation disabled, use plain pointers + +template +struct GenericMemberIterator; + +//! non-const GenericMemberIterator +template +struct GenericMemberIterator { + //! use plain pointer as iterator type + typedef GenericMember* Iterator; +}; +//! const GenericMemberIterator +template +struct GenericMemberIterator { + //! use plain const pointer as iterator type + typedef const GenericMember* Iterator; +}; + +#endif // RAPIDJSON_NOMEMBERITERATORCLASS + +/////////////////////////////////////////////////////////////////////////////// +// GenericStringRef + +//! Reference to a constant string (not taking a copy) +/*! + \tparam CharType character type of the string + + This helper class is used to automatically infer constant string + references for string literals, especially from \c const \b (!) + character arrays. + + The main use is for creating JSON string values without copying the + source string via an \ref Allocator. This requires that the referenced + string pointers have a sufficient lifetime, which exceeds the lifetime + of the associated GenericValue. + + \b Example + \code + Value v("foo"); // ok, no need to copy & calculate length + const char foo[] = "foo"; + v.SetString(foo); // ok + + const char* bar = foo; + // Value x(bar); // not ok, can't rely on bar's lifetime + Value x(StringRef(bar)); // lifetime explicitly guaranteed by user + Value y(StringRef(bar, 3)); // ok, explicitly pass length + \endcode + + \see StringRef, GenericValue::SetString +*/ +template +struct GenericStringRef { + typedef CharType Ch; //!< character type of the string + + //! Create string reference from \c const character array +#ifndef __clang__ // -Wdocumentation + /*! + This constructor implicitly creates a constant string reference from + a \c const character array. It has better performance than + \ref StringRef(const CharType*) by inferring the string \ref length + from the array length, and also supports strings containing null + characters. + + \tparam N length of the string, automatically inferred + + \param str Constant character array, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note Constant complexity. + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ +#endif + template + GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT + : s(str), length(N-1) {} + + //! Explicitly create string reference from \c const character pointer +#ifndef __clang__ // -Wdocumentation + /*! + This constructor can be used to \b explicitly create a reference to + a constant string pointer. + + \see StringRef(const CharType*) + + \param str Constant character pointer, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ +#endif + explicit GenericStringRef(const CharType* str) + : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != 0); } + + //! Create constant string reference from pointer and length +#ifndef __clang__ // -Wdocumentation + /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param len length of the string, excluding the trailing NULL terminator + + \post \ref s == str && \ref length == len + \note Constant complexity. + */ +#endif + GenericStringRef(const CharType* str, SizeType len) + : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } + + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} + + GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } + + //! implicit conversion to plain CharType pointer + operator const Ch *() const { return s; } + + const Ch* const s; //!< plain CharType pointer + const SizeType length; //!< length of the string (excluding the trailing NULL terminator) + +private: + //! Disallow construction from non-const array + template + GenericStringRef(CharType (&str)[N]) /* = delete */; +}; + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + \tparam CharType Character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + + \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember +*/ +template +inline GenericStringRef StringRef(const CharType* str) { + return GenericStringRef(str, internal::StrLen(str)); +} + +//! Mark a character pointer as constant string +/*! Mark a plain character pointer as a "string literal". This function + can be used to avoid copying a character string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + This version has better performance with supplied length, and also + supports string containing null characters. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param length The length of source string. + \return GenericStringRef string reference object + \relatesalso GenericStringRef +*/ +template +inline GenericStringRef StringRef(const CharType* str, size_t length) { + return GenericStringRef(str, SizeType(length)); +} + +#if RAPIDJSON_HAS_STDSTRING +//! Mark a string object as constant string +/*! Mark a string object (e.g. \c std::string) as a "string literal". + This function can be used to avoid copying a string to be referenced as a + value in a JSON GenericValue object, if the string's lifetime is known + to be valid long enough. + + \tparam CharType character type of the string + \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \return GenericStringRef string reference object + \relatesalso GenericStringRef + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. +*/ +template +inline GenericStringRef StringRef(const std::basic_string& str) { + return GenericStringRef(str.data(), SizeType(str.size())); +} +#endif + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue type traits +namespace internal { + +template +struct IsGenericValueImpl : FalseType {}; + +// select candidates according to nested encoding and allocator types +template struct IsGenericValueImpl::Type, typename Void::Type> + : IsBaseOf, T>::Type {}; + +// helper to match arbitrary GenericValue instantiations, including derived classes +template struct IsGenericValue : IsGenericValueImpl::Type {}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// TypeHelper + +namespace internal { + +template +struct TypeHelper {}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsBool(); } + static bool Get(const ValueType& v) { return v.GetBool(); } + static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } + static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static int Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt64(); } + static int64_t Get(const ValueType& v) { return v.GetInt64(); } + static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } + static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint64(); } + static uint64_t Get(const ValueType& v) { return v.GetUint64(); } + static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } + static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsDouble(); } + static double Get(const ValueType& v) { return v.GetDouble(); } + static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } + static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } +}; + +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsFloat(); } + static float Get(const ValueType& v) { return v.GetFloat(); } + static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } + static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } +}; + +template +struct TypeHelper { + typedef const typename ValueType::Ch* StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } + static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; + +#if RAPIDJSON_HAS_STDSTRING +template +struct TypeHelper > { + typedef std::basic_string StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +}; +#endif + +template +struct TypeHelper { + typedef typename ValueType::Array ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(ValueType& v) { return v.GetArray(); } + static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } + static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstArray ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(const ValueType& v) { return v.GetArray(); } +}; + +template +struct TypeHelper { + typedef typename ValueType::Object ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(ValueType& v) { return v.GetObject(); } + static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } +}; + +template +struct TypeHelper { + typedef typename ValueType::ConstObject ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(const ValueType& v) { return v.GetObject(); } +}; + +} // namespace internal + +// Forward declarations +template class GenericArray; +template class GenericObject; + +/////////////////////////////////////////////////////////////////////////////// +// GenericValue + +//! Represents a JSON value. Use Value for UTF8 encoding and default allocator. +/*! + A JSON value can be one of 7 types. This class is a variant type supporting + these types. + + Use the Value if UTF8 and default allocator + + \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) + \tparam Allocator Allocator type for allocating memory of object, array and string. +*/ +template > +class GenericValue { +public: + //! Name-value pair in an object. + typedef GenericMember Member; + typedef Encoding EncodingType; //!< Encoding type from template parameter. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericStringRef StringRefType; //!< Reference to a constant string + typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. + typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. + typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. + typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. + typedef GenericValue ValueType; //!< Value type of itself. + typedef GenericArray Array; + typedef GenericArray ConstArray; + typedef GenericObject Object; + typedef GenericObject ConstObject; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor creates a null value. + GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { + rhs.data_.f.flags = kNullFlag; // give up contents + } +#endif + +private: + //! Copy constructor is not permitted. + GenericValue(const GenericValue& rhs); + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Moving from a GenericDocument is not permitted. + template + GenericValue(GenericDocument&& rhs); + + //! Move assignment from a GenericDocument is not permitted. + template + GenericValue& operator=(GenericDocument&& rhs); +#endif + +public: + + //! Constructor with JSON value type. + /*! This creates a Value of specified type with default content. + \param type Type of the value. + \note Default content for number is zero. + */ + explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { + static const uint16_t defaultFlags[7] = { + kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, + kNumberAnyFlag + }; + RAPIDJSON_ASSERT(type <= kNumberType); + data_.f.flags = defaultFlags[type]; + + // Use ShortString to store empty string. + if (type == kStringType) + data_.ss.SetLength(0); + } + + //! Explicit copy constructor (with allocator) + /*! Creates a copy of a Value by using the given Allocator + \tparam SourceAllocator allocator of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \see CopyFrom() + */ + template< typename SourceAllocator > + GenericValue(const GenericValue& rhs, Allocator & allocator); + + //! Constructor for boolean value. + /*! \param b Boolean value + \note This constructor is limited to \em real boolean values and rejects + implicitly converted types like arbitrary pointers. Use an explicit cast + to \c bool, if you want to construct a boolean JSON value in such cases. + */ +#ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen + template + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 +#else + explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT +#endif + : data_() { + // safe-guard against failing SFINAE + RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + data_.f.flags = b ? kTrueFlag : kFalseFlag; + } + + //! Constructor for int value. + explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { + data_.n.i64 = i; + data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; + } + + //! Constructor for unsigned value. + explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { + data_.n.u64 = u; + data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); + } + + //! Constructor for int64_t value. + explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { + data_.n.i64 = i64; + data_.f.flags = kNumberInt64Flag; + if (i64 >= 0) { + data_.f.flags |= kNumberUint64Flag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for uint64_t value. + explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { + data_.n.u64 = u64; + data_.f.flags = kNumberUint64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) + data_.f.flags |= kInt64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for double value. + explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for constant string (i.e. do not make a copy of string) + GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } + + //! Constructor for constant string (i.e. do not make a copy of string) + explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } + +#if RAPIDJSON_HAS_STDSTRING + //! Constructor for copy-string from a string object (i.e. do make a copy of string) + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } +#endif + + //! Constructor for Array. + /*! + \param a An array obtained by \c GetArray(). + \note \c Array is always pass-by-value. + \note the source array is moved into this value and the sourec array becomes empty. + */ + GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { + a.value_.data_ = Data(); + a.value_.data_.f.flags = kArrayFlag; + } + + //! Constructor for Object. + /*! + \param o An object obtained by \c GetObject(). + \note \c Object is always pass-by-value. + \note the source object is moved into this value and the sourec object becomes empty. + */ + GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { + o.value_.data_ = Data(); + o.value_.data_.f.flags = kObjectFlag; + } + + //! Destructor. + /*! Need to destruct elements of array, members of object, or copy-string. + */ + ~GenericValue() { + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + switch(data_.f.flags) { + case kArrayFlag: + { + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(e); + } + break; + + case kObjectFlag: + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); + break; + + case kCopyStringFlag: + Allocator::Free(const_cast(GetStringPointer())); + break; + + default: + break; // Do nothing for other types. + } + } + } + + //@} + + //!@name Assignment operators + //@{ + + //! Assignment with move semantics. + /*! \param rhs Source of the assignment. It will become a null value after assignment. + */ + GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { + RAPIDJSON_ASSERT(this != &rhs); + this->~GenericValue(); + RawAssign(rhs); + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { + return *this = rhs.Move(); + } +#endif + + //! Assignment of constant string reference (no copy) + /*! \param str Constant string reference to be assigned + \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. + \see GenericStringRef, operator=(T) + */ + GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { + GenericValue s(str); + return *this = s; + } + + //! Assignment with primitive types. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value The value to be assigned. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref SetString(const Ch*, Allocator&) (for copying) or + \ref StringRef() (to explicitly mark the pointer as constant) instead. + All other pointer types would implicitly convert to \c bool, + use \ref SetBool() instead. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) + operator=(T value) { + GenericValue v(value); + return *this = v; + } + + //! Deep-copy assignment from Value + /*! Assigns a \b copy of the Value to the current Value object + \tparam SourceAllocator Allocator type of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator to use for copying + */ + template + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { + RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); + this->~GenericValue(); + new (this) GenericValue(rhs, allocator); + return *this; + } + + //! Exchange the contents of this value with those of other. + /*! + \param other Another value. + \note Constant complexity. + */ + GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { + GenericValue temp; + temp.RawAssign(*this); + RawAssign(other); + other.RawAssign(temp); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.value, b.value); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Prepare Value for move semantics + /*! \return *this */ + GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } + //@} + + //!@name Equal-to and not-equal-to operators + //@{ + //! Equal-to operator + /*! + \note If an object contains duplicated named member, comparing equality with any object is always \c false. + \note Linear time complexity (number of all values in the subtree and total lengths of all strings). + */ + template + bool operator==(const GenericValue& rhs) const { + typedef GenericValue RhsType; + if (GetType() != rhs.GetType()) + return false; + + switch (GetType()) { + case kObjectType: // Warning: O(n^2) inner-loop + if (data_.o.size != rhs.data_.o.size) + return false; + for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { + typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); + if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) + return false; + } + return true; + + case kArrayType: + if (data_.a.size != rhs.data_.a.size) + return false; + for (SizeType i = 0; i < data_.a.size; i++) + if ((*this)[i] != rhs[i]) + return false; + return true; + + case kStringType: + return StringEqual(rhs); + + case kNumberType: + if (IsDouble() || rhs.IsDouble()) { + double a = GetDouble(); // May convert from integer to double. + double b = rhs.GetDouble(); // Ditto + return a >= b && a <= b; // Prevent -Wfloat-equal + } + else + return data_.n.u64 == rhs.data_.n.u64; + + default: + return true; + } + } + + //! Equal-to operator with const C-string pointer + bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } + +#if RAPIDJSON_HAS_STDSTRING + //! Equal-to operator with string object + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } +#endif + + //! Equal-to operator with primitive types + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } + + //! Not-equal-to operator + /*! \return !(*this == rhs) + */ + template + bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with const C-string pointer + bool operator!=(const Ch* rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with arbitrary types + /*! \return !(*this == rhs) + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } + + //! Equal-to operator with arbitrary types (symmetric version) + /*! \return (rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } + + //! Not-Equal-to operator with arbitrary types (symmetric version) + /*! \return !(rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } + //@} + + //!@name Type + //@{ + + Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } + bool IsNull() const { return data_.f.flags == kNullFlag; } + bool IsFalse() const { return data_.f.flags == kFalseFlag; } + bool IsTrue() const { return data_.f.flags == kTrueFlag; } + bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } + bool IsObject() const { return data_.f.flags == kObjectFlag; } + bool IsArray() const { return data_.f.flags == kArrayFlag; } + bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } + bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } + bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } + bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } + bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } + bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } + bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } + + // Checks whether a number can be losslessly converted to a double. + bool IsLosslessDouble() const { + if (!IsNumber()) return false; + if (IsUint64()) { + uint64_t u = GetUint64(); + volatile double d = static_cast(u); + return (d >= 0.0) + && (d < static_cast(std::numeric_limits::max())) + && (u == static_cast(d)); + } + if (IsInt64()) { + int64_t i = GetInt64(); + volatile double d = static_cast(i); + return (d >= static_cast(std::numeric_limits::min())) + && (d < static_cast(std::numeric_limits::max())) + && (i == static_cast(d)); + } + return true; // double, int, uint are always lossless + } + + // Checks whether a number is a float (possible lossy). + bool IsFloat() const { + if ((data_.f.flags & kDoubleFlag) == 0) + return false; + double d = GetDouble(); + return d >= -3.4028234e38 && d <= 3.4028234e38; + } + // Checks whether a number can be losslessly converted to a float. + bool IsLosslessFloat() const { + if (!IsNumber()) return false; + double a = GetDouble(); + if (a < static_cast(-std::numeric_limits::max()) + || a > static_cast(std::numeric_limits::max())) + return false; + double b = static_cast(static_cast(a)); + return a >= b && a <= b; // Prevent -Wfloat-equal + } + + //@} + + //!@name Null + //@{ + + GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } + + //@} + + //!@name Bool + //@{ + + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } + //!< Set boolean value + /*! \post IsBool() == true */ + GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } + + //@} + + //!@name Object + //@{ + + //! Set this value as an empty object. + /*! \post IsObject() == true */ + GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } + + //! Get the number of members in the object. + SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } + + //! Check whether the object is empty. + bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) + \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. + Since 0.2, if the name is not correct, it will assert. + If user is unsure whether a member exists, user should use HasMember() first. + A better approach is to use FindMember(). + \note Linear time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { + GenericValue n(StringRef(name)); + return (*this)[n]; + } + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam SourceAllocator Allocator of the \c name value + + \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). + And it can also handle strings with embedded null characters. + + \note Linear time complexity. + */ + template + GenericValue& operator[](const GenericValue& name) { + MemberIterator member = FindMember(name); + if (member != MemberEnd()) + return member->value; + else { + RAPIDJSON_ASSERT(false); // see above note + + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); + } + } + template + const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } + +#if RAPIDJSON_HAS_STDSTRING + //! Get a value from an object associated with name (string object). + GenericValue& operator[](const std::basic_string& name) { return (*this)[GenericValue(StringRef(name))]; } + const GenericValue& operator[](const std::basic_string& name) const { return (*this)[GenericValue(StringRef(name))]; } +#endif + + //! Const member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } + //! Const \em past-the-end member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } + //! Member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } + //! \em Past-the-end member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } + + //! Check whether a member exists in the object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } + +#if RAPIDJSON_HAS_STDSTRING + //! Check whether a member exists in the object with string object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const std::basic_string& name) const { return FindMember(name) != MemberEnd(); } +#endif + + //! Check whether a member exists in the object with GenericValue name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + template + bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } + + //! Find member by name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + MemberIterator FindMember(const Ch* name) { + GenericValue n(StringRef(name)); + return FindMember(n); + } + + ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } + + //! Find member by name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + template + MemberIterator FindMember(const GenericValue& name) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } + +#if RAPIDJSON_HAS_STDSTRING + //! Find member by string object name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + */ + MemberIterator FindMember(const std::basic_string& name) { return FindMember(StringRef(name)); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(StringRef(name)); } +#endif + + //! Add a member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c name and \c value will be transferred to this object on success. + \pre IsObject() && name.IsString() + \post name.IsNull() && value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + + ObjectData& o = data_.o; + if (o.size >= o.capacity) { + if (o.capacity == 0) { + o.capacity = kDefaultObjectCapacity; + SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); + } + else { + SizeType oldCapacity = o.capacity; + o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); + } + } + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); + o.size++; + return *this; + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Add a string object as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, std::basic_string& value, Allocator& allocator) { + GenericValue v(value, allocator); + return AddMember(name, v, allocator); + } +#endif + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A string value as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(GenericValue& name, T value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { + return AddMember(name, value, allocator); + } + GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + + + //! Add a member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this object on success. + \pre IsObject() + \post value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { + GenericValue v(value); + return AddMember(name, v, allocator); + } + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A constant string reference as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(StringRefType name, T value, Allocator& allocator) { + GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Remove all members in the object. + /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void RemoveAllMembers() { + RAPIDJSON_ASSERT(IsObject()); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + //! Remove a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Linear time complexity. + */ + bool RemoveMember(const Ch* name) { + GenericValue n(StringRef(name)); + return RemoveMember(n); + } + +#if RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) { return RemoveMember(GenericValue(StringRef(name))); } +#endif + + template + bool RemoveMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + RemoveMember(m); + return true; + } + else + return false; + } + + //! Remove a member in object by iterator. + /*! \param m member iterator (obtained by FindMember() or MemberBegin()). + \return the new iterator after removal. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Constant time complexity. + */ + MemberIterator RemoveMember(MemberIterator m) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); + + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy + --data_.o.size; + return m; + } + + //! Remove a member from an object by iterator. + /*! \param pos iterator to the member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() + \return Iterator following the removed element. + If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. + \note This function preserves the relative order of the remaining object + members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator pos) { + return EraseMember(pos, pos +1); + } + + //! Remove members in the range [first, last) from an object. + /*! \param first iterator to the first member to remove + \param last iterator following the last member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() + \return Iterator following the last removed element. + \note This function preserves the relative order of the remaining object + members. + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { + RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(first >= MemberBegin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= MemberEnd()); + + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); + std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); + return pos; + } + + //! Erase a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note Linear time complexity. + */ + bool EraseMember(const Ch* name) { + GenericValue n(StringRef(name)); + return EraseMember(n); + } + +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } +#endif + + template + bool EraseMember(const GenericValue& name) { + MemberIterator m = FindMember(name); + if (m != MemberEnd()) { + EraseMember(m); + return true; + } + else + return false; + } + + Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + + //@} + + //!@name Array + //@{ + + //! Set this value as an empty array. + /*! \post IsArray == true */ + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + + //! Get the number of elements in array. + SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } + + //! Get the capacity of array. + SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } + + //! Check whether the array is empty. + bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } + + //! Remove all elements in the array. + /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void Clear() { + RAPIDJSON_ASSERT(IsArray()); + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + data_.a.size = 0; + } + + //! Get an element from array by index. + /*! \pre IsArray() == true + \param index Zero-based index of element. + \see operator[](T*) + */ + GenericValue& operator[](SizeType index) { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(index < data_.a.size); + return GetElementsPointer()[index]; + } + const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } + + //! Element iterator + /*! \pre IsArray() == true */ + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } + //! \em Past-the-end element iterator + /*! \pre IsArray() == true */ + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } + //! Constant element iterator + /*! \pre IsArray() == true */ + ConstValueIterator Begin() const { return const_cast(*this).Begin(); } + //! Constant \em past-the-end element iterator + /*! \pre IsArray() == true */ + ConstValueIterator End() const { return const_cast(*this).End(); } + + //! Request the array to have enough capacity to store elements. + /*! \param newCapacity The capacity that the array at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { + RAPIDJSON_ASSERT(IsArray()); + if (newCapacity > data_.a.capacity) { + SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); + data_.a.capacity = newCapacity; + } + return *this; + } + + //! Append a GenericValue at the end of the array. + /*! \param value Value to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \post value.IsNull() == true + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this array on success. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + */ + GenericValue& PushBack(GenericValue& value, Allocator& allocator) { + RAPIDJSON_ASSERT(IsArray()); + if (data_.a.size >= data_.a.capacity) + Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); + GetElementsPointer()[data_.a.size++].RawAssign(value); + return *this; + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { + return PushBack(value, allocator); + } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + + //! Append a constant string reference at the end of the array. + /*! \param value Constant string reference to be appended. + \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + \see GenericStringRef + */ + GenericValue& PushBack(StringRefType value, Allocator& allocator) { + return (*this).template PushBack(value, allocator); + } + + //! Append a primitive value at the end of the array. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value Value of primitive type T to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref PushBack(GenericValue&, Allocator&) or \ref + PushBack(StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + PushBack(T value, Allocator& allocator) { + GenericValue v(value); + return PushBack(v, allocator); + } + + //! Remove the last element in the array. + /*! + \note Constant time complexity. + */ + GenericValue& PopBack() { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(!Empty()); + GetElementsPointer()[--data_.a.size].~GenericValue(); + return *this; + } + + //! Remove an element of array by iterator. + /*! + \param pos iterator to the element to remove + \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() + \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator pos) { + return Erase(pos, pos + 1); + } + + //! Remove elements in the range [first, last) of the array. + /*! + \param first iterator to the first element to remove + \param last iterator following the last element to remove + \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() + \return Iterator following the last removed element. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { + RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(data_.a.size > 0); + RAPIDJSON_ASSERT(GetElementsPointer() != 0); + RAPIDJSON_ASSERT(first >= Begin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= End()); + ValueIterator pos = Begin() + (first - Begin()); + for (ValueIterator itr = pos; itr != last; ++itr) + itr->~GenericValue(); + std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); + return pos; + } + + Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } + ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } + + //@} + + //!@name Number + //@{ + + int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + + //! Get the value as double type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. + */ + double GetDouble() const { + RAPIDJSON_ASSERT(IsNumber()); + if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) + } + + //! Get the value as float type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. + */ + float GetFloat() const { + return static_cast(GetDouble()); + } + + GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } + GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } + GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } + GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } + GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } + + //@} + + //!@name String + //@{ + + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + + //! Get the length of string. + /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). + */ + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + + //! Set this value as a string without copying source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string pointer. + \param length The length of source string, excluding the trailing null terminator. + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == length + \see SetString(StringRefType) + */ + GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } + + //! Set this value as a string without copying source string. + /*! \param s source string reference + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == s.length + */ + GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } + + //! Set this value as a string by copying from source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string. + \param length The length of source string, excluding the trailing null terminator. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + +#if RAPIDJSON_HAS_STDSTRING + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } +#endif + + //@} + + //!@name Array + //@{ + + //! Templated version for checking whether this value is type T. + /*! + \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string + */ + template + bool Is() const { return internal::TypeHelper::Is(*this); } + + template + T Get() const { return internal::TypeHelper::Get(*this); } + + template + T Get() { return internal::TypeHelper::Get(*this); } + + template + ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } + + template + ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } + + //@} + + //! Generate events of this value to a Handler. + /*! This function adopts the GoF visitor pattern. + Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. + It can also be used to deep clone this value via GenericDocument, which is also a Handler. + \tparam Handler type of handler. + \param handler An object implementing concept Handler. + */ + template + bool Accept(Handler& handler) const { + switch(GetType()) { + case kNullType: return handler.Null(); + case kFalseType: return handler.Bool(false); + case kTrueType: return handler.Bool(true); + + case kObjectType: + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + return false; + for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { + RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. + if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) + return false; + if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) + return false; + } + return handler.EndObject(data_.o.size); + + case kArrayType: + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + return false; + for (const GenericValue* v = Begin(); v != End(); ++v) + if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) + return false; + return handler.EndArray(data_.a.size); + + case kStringType: + return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); + + default: + RAPIDJSON_ASSERT(GetType() == kNumberType); + if (IsDouble()) return handler.Double(data_.n.d); + else if (IsInt()) return handler.Int(data_.n.i.i); + else if (IsUint()) return handler.Uint(data_.n.u.u); + else if (IsInt64()) return handler.Int64(data_.n.i64); + else return handler.Uint64(data_.n.u64); + } + } + +private: + template friend class GenericValue; + template friend class GenericDocument; + + enum { + kBoolFlag = 0x0008, + kNumberFlag = 0x0010, + kIntFlag = 0x0020, + kUintFlag = 0x0040, + kInt64Flag = 0x0080, + kUint64Flag = 0x0100, + kDoubleFlag = 0x0200, + kStringFlag = 0x0400, + kCopyFlag = 0x0800, + kInlineStrFlag = 0x1000, + + // Initial flags of different types. + kNullFlag = kNullType, + kTrueFlag = kTrueType | kBoolFlag, + kFalseFlag = kFalseType | kBoolFlag, + kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, + kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, + kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, + kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, + kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, + kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, + kConstStringFlag = kStringType | kStringFlag, + kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, + kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + kObjectFlag = kObjectType, + kArrayFlag = kArrayType, + + kTypeMask = 0x07 + }; + + static const SizeType kDefaultArrayCapacity = 16; + static const SizeType kDefaultObjectCapacity = 16; + + struct Flag { +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION + char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer +#elif RAPIDJSON_64BIT + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes +#else + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes +#endif + uint16_t flags; + }; + + struct String { + SizeType length; + SizeType hashcode; //!< reserved + const Ch* str; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars + // (excluding the terminating zero) and store a value to determine the length of the contained + // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string + // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as + // the string terminator as well. For getting the string length back from that value just use + // "MaxSize - str[LenPos]". + // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, + // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). + struct ShortString { + enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + Ch str[MaxChars]; + + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } + inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } + }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // By using proper binary layout, retrieval of different integer types do not need conversions. + union Number { +#if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN + struct I { + int i; + char padding[4]; + }i; + struct U { + unsigned u; + char padding2[4]; + }u; +#else + struct I { + char padding[4]; + int i; + }i; + struct U { + char padding2[4]; + unsigned u; + }u; +#endif + int64_t i64; + uint64_t u64; + double d; + }; // 8 bytes + + struct ObjectData { + SizeType size; + SizeType capacity; + Member* members; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + struct ArrayData { + SizeType size; + SizeType capacity; + GenericValue* elements; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + union Data { + String s; + ShortString ss; + Number n; + ObjectData o; + ArrayData a; + Flag f; + }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } + RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } + RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } + RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } + RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } + RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } + + // Initialize this value as array with initial data, without calling destructor. + void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { + data_.f.flags = kArrayFlag; + if (count) { + GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); + std::memcpy(e, values, count * sizeof(GenericValue)); + } + else + SetElementsPointer(0); + data_.a.size = data_.a.capacity = count; + } + + //! Initialize this value as object with initial data, without calling destructor. + void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { + data_.f.flags = kObjectFlag; + if (count) { + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); + std::memcpy(m, members, count * sizeof(Member)); + } + else + SetMembersPointer(0); + data_.o.size = data_.o.capacity = count; + } + + //! Initialize this value as constant string, without calling destructor. + void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { + data_.f.flags = kConstStringFlag; + SetStringPointer(s); + data_.s.length = s.length; + } + + //! Initialize this value as copy string with initial data, without calling destructor. + void SetStringRaw(StringRefType s, Allocator& allocator) { + Ch* str = 0; + if (ShortString::Usable(s.length)) { + data_.f.flags = kShortStringFlag; + data_.ss.SetLength(s.length); + str = data_.ss.str; + } else { + data_.f.flags = kCopyStringFlag; + data_.s.length = s.length; + str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); + SetStringPointer(str); + } + std::memcpy(str, s, s.length * sizeof(Ch)); + str[s.length] = '\0'; + } + + //! Assignment without calling destructor + void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { + data_ = rhs.data_; + // data_.f.flags = rhs.data_.f.flags; + rhs.data_.f.flags = kNullFlag; + } + + template + bool StringEqual(const GenericValue& rhs) const { + RAPIDJSON_ASSERT(IsString()); + RAPIDJSON_ASSERT(rhs.IsString()); + + const SizeType len1 = GetStringLength(); + const SizeType len2 = rhs.GetStringLength(); + if(len1 != len2) { return false; } + + const Ch* const str1 = GetString(); + const Ch* const str2 = rhs.GetString(); + if(str1 == str2) { return true; } // fast path for constant string + + return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); + } + + Data data_; +}; + +//! GenericValue with UTF8 encoding +typedef GenericValue > Value; + +/////////////////////////////////////////////////////////////////////////////// +// GenericDocument + +//! A document for parsing JSON text as DOM. +/*! + \note implements Handler concept + \tparam Encoding Encoding for both parsing and string storage. + \tparam Allocator Allocator for allocating memory for the DOM + \tparam StackAllocator Allocator for allocating memory for stack during parsing. + \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. +*/ +template , typename StackAllocator = CrtAllocator> +class GenericDocument : public GenericValue { +public: + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericValue ValueType; //!< Value type of the document. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + + //! Constructor + /*! Creates an empty document of specified type. + \param type Mandatory type of object to create. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + } + + //! Constructor + /*! Creates an empty document which type is Null. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(std::move(rhs.stack_)), + parseResult_(rhs.parseResult_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + } +#endif + + ~GenericDocument() { + Destroy(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + { + // The cast to ValueType is necessary here, because otherwise it would + // attempt to call GenericValue's templated assignment operator. + ValueType::operator=(std::forward(rhs)); + + // Calling the destructor here would prematurely call stack_'s destructor + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = std::move(rhs.stack_); + parseResult_ = rhs.parseResult_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + + return *this; + } +#endif + + //! Exchange the contents of this document with those of another. + /*! + \param rhs Another document. + \note Constant complexity. + \see GenericValue::Swap + */ + GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { + ValueType::Swap(rhs); + stack_.Swap(rhs.stack_); + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(parseResult_, rhs.parseResult_); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.doc, b.doc); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Populate this document by a generator which produces SAX events. + /*! \tparam Generator A functor with bool f(Handler) prototype. + \param g Generator functor which sends SAX events to the parameter. + \return The document itself for fluent API. + */ + template + GenericDocument& Populate(Generator& g) { + ClearStackOnExit scope(*this); + if (g(*this)) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //!@name Parse from stream + //!@{ + + //! Parse JSON text from an input stream (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam SourceEncoding Encoding of input stream + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + GenericReader reader( + stack_.HasAllocator() ? &stack_.GetAllocator() : 0); + ClearStackOnExit scope(*this); + parseResult_ = reader.template Parse(is, *this); + if (parseResult_) { + RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //! Parse JSON text from an input stream + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + + //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) { + return ParseStream(is); + } + //!@} + + //!@name Parse in-place from mutable string + //!@{ + + //! Parse JSON text from a mutable string + /*! \tparam parseFlags Combination of \ref ParseFlag. + \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseInsitu(Ch* str) { + GenericInsituStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) + /*! \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + GenericDocument& ParseInsitu(Ch* str) { + return ParseInsitu(str); + } + //!@} + + //!@name Parse from read-only string + //!@{ + + //! Parse JSON text from a read-only string (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \tparam SourceEncoding Transcoding from input Encoding + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + GenericStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a read-only string + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) + /*! \param str Read-only zero-terminated string to be parsed. + */ + GenericDocument& Parse(const Ch* str) { + return Parse(str); + } + + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { + RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + MemoryStream ms(static_cast(str), length * sizeof(typename SourceEncoding::Ch)); + EncodedInputStream is(ms); + ParseStream(is); + return *this; + } + + template + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + + GenericDocument& Parse(const Ch* str, size_t length) { + return Parse(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + template + GenericDocument& Parse(const std::basic_string& str) { + // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) + return Parse(str.c_str()); + } + + template + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str.c_str()); + } + + GenericDocument& Parse(const std::basic_string& str) { + return Parse(str); + } +#endif // RAPIDJSON_HAS_STDSTRING + + //!@} + + //!@name Handling parse errors + //!@{ + + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseError() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + + //! Implicit conversion to get the last parse result +#ifndef __clang // -Wdocumentation + /*! \return \ref ParseResult of the last parse operation + + \code + Document doc; + ParseResult ok = doc.Parse(json); + if (!ok) + printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); + \endcode + */ +#endif + operator ParseResult() const { return parseResult_; } + //!@} + + //! Get the allocator of this document. + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + //! Get the capacity of stack in bytes. + size_t GetStackCapacity() const { return stack_.GetCapacity(); } + +private: + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} + ~ClearStackOnExit() { d_.ClearStack(); } + private: + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + GenericDocument& d_; + }; + + // callers of the following private Handler functions + // template friend class GenericReader; // for parsing + template friend class GenericValue; // for deep copying + +public: + // Implementation of Handler + bool Null() { new (stack_.template Push()) ValueType(); return true; } + bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } + bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } + bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + + bool RawNumber(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool String(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } + + bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount) { + typename ValueType::Member* members = stack_.template Pop(memberCount); + stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); + return true; + } + + bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } + + bool EndArray(SizeType elementCount) { + ValueType* elements = stack_.template Pop(elementCount); + stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); + return true; + } + +private: + //! Prohibit copying + GenericDocument(const GenericDocument&); + //! Prohibit assignment + GenericDocument& operator=(const GenericDocument&); + + void ClearStack() { + if (Allocator::kNeedFree) + while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) + (stack_.template Pop(1))->~ValueType(); + else + stack_.Clear(); + stack_.ShrinkToFit(); + } + + void Destroy() { + RAPIDJSON_DELETE(ownAllocator_); + } + + static const size_t kDefaultStackCapacity = 1024; + Allocator* allocator_; + Allocator* ownAllocator_; + internal::Stack stack_; + ParseResult parseResult_; +}; + +//! GenericDocument with UTF8 encoding +typedef GenericDocument > Document; + +// defined here due to the dependency on GenericDocument +template +template +inline +GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) +{ + switch (rhs.GetType()) { + case kObjectType: + case kArrayType: { // perform deep copy via SAX Handler + GenericDocument d(&allocator); + rhs.Accept(d); + RawAssign(*d.stack_.template Pop(1)); + } + break; + case kStringType: + if (rhs.data_.f.flags == kConstStringFlag) { + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + } else { + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + } + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } +} + +//! Helper class for accessing Value of array type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetArray(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericArray { +public: + typedef GenericArray ConstArray; + typedef GenericArray Array; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef ValueType* ValueIterator; // This may be const or non-const iterator + typedef const ValueT* ConstValueIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + + template + friend class GenericValue; + + GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} + GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } + ~GenericArray() {} + + SizeType Size() const { return value_.Size(); } + SizeType Capacity() const { return value_.Capacity(); } + bool Empty() const { return value_.Empty(); } + void Clear() const { value_.Clear(); } + ValueType& operator[](SizeType index) const { return value_[index]; } + ValueIterator Begin() const { return value_.Begin(); } + ValueIterator End() const { return value_.End(); } + GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } + GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PopBack() const { value_.PopBack(); return *this; } + ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + ValueIterator begin() const { return value_.Begin(); } + ValueIterator end() const { return value_.End(); } +#endif + +private: + GenericArray(); + GenericArray(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +//! Helper class for accessing Value of object type. +/*! + Instance of this helper class is obtained by \c GenericValue::GetObject(). + In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. +*/ +template +class GenericObject { +public: + typedef GenericObject ConstObject; + typedef GenericObject Object; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator + typedef GenericMemberIterator ConstMemberIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename ValueType::Ch Ch; + + template + friend class GenericValue; + + GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} + GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } + ~GenericObject() {} + + SizeType MemberCount() const { return value_.MemberCount(); } + bool ObjectEmpty() const { return value_.ObjectEmpty(); } + template ValueType& operator[](T* name) const { return value_[name]; } + template ValueType& operator[](const GenericValue& name) const { return value_[name]; } +#if RAPIDJSON_HAS_STDSTRING + ValueType& operator[](const std::basic_string& name) const { return value_[name]; } +#endif + MemberIterator MemberBegin() const { return value_.MemberBegin(); } + MemberIterator MemberEnd() const { return value_.MemberEnd(); } + bool HasMember(const Ch* name) const { return value_.HasMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } +#endif + template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } + MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } + template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } +#if RAPIDJSON_HAS_STDSTRING + MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } +#endif + GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_STDSTRING + GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + void RemoveAllMembers() { return value_.RemoveAllMembers(); } + bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } +#endif + template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } + MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } + MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } + bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } +#if RAPIDJSON_HAS_STDSTRING + bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } +#endif + template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } + +#if RAPIDJSON_HAS_CXX11_RANGE_FOR + MemberIterator begin() const { return value_.MemberBegin(); } + MemberIterator end() const { return value_.MemberEnd(); } +#endif + +private: + GenericObject(); + GenericObject(ValueType& value) : value_(value) {} + ValueType& value_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_DOCUMENT_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/encodedstream.h b/ext/librethinkdbxx/src/rapidjson/encodedstream.h new file mode 100644 index 00000000..14506838 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/encodedstream.h @@ -0,0 +1,299 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ENCODEDSTREAM_H_ +#define RAPIDJSON_ENCODEDSTREAM_H_ + +#include "stream.h" +#include "memorystream.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Input byte stream wrapper with a statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam InputByteStream Type of input byte stream. For example, FileReadStream. +*/ +template +class EncodedInputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedInputStream(InputByteStream& is) : is_(is) { + current_ = Encoding::TakeBOM(is_); + } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); + + InputByteStream& is_; + Ch current_; +}; + +//! Specialized for UTF8 MemoryStream. +template <> +class EncodedInputStream, MemoryStream> { +public: + typedef UTF8<>::Ch Ch; + + EncodedInputStream(MemoryStream& is) : is_(is) { + if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); + } + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() const { return is_.Tell(); } + + // Not implemented + void Put(Ch) {} + void Flush() {} + Ch* PutBegin() { return 0; } + size_t PutEnd(Ch*) { return 0; } + + MemoryStream& is_; + +private: + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); +}; + +//! Output byte stream wrapper with statically bound encoding. +/*! + \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. + \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. +*/ +template +class EncodedOutputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef typename Encoding::Ch Ch; + + EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { + if (putBOM) + Encoding::PutBOM(os_); + } + + void Put(Ch c) { Encoding::Put(os_, c); } + void Flush() { os_.Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + EncodedOutputStream(const EncodedOutputStream&); + EncodedOutputStream& operator=(const EncodedOutputStream&); + + OutputByteStream& os_; +}; + +#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + +//! Input stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for reading. + \tparam InputByteStream type of input byte stream to be wrapped. +*/ +template +class AutoUTFInputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param is input stream to be wrapped. + \param type UTF encoding type if it is not detected from the stream. + */ + AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { + RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + DetectType(); + static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; + takeFunc_ = f[type_]; + current_ = takeFunc_(*is_); + } + + UTFType GetType() const { return type_; } + bool HasBOM() const { return hasBOM_; } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } + size_t Tell() const { return is_->Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFInputStream(const AutoUTFInputStream&); + AutoUTFInputStream& operator=(const AutoUTFInputStream&); + + // Detect encoding type with BOM or RFC 4627 + void DetectType() { + // BOM (Byte Order Mark): + // 00 00 FE FF UTF-32BE + // FF FE 00 00 UTF-32LE + // FE FF UTF-16BE + // FF FE UTF-16LE + // EF BB BF UTF-8 + + const unsigned char* c = reinterpret_cast(is_->Peek4()); + if (!c) + return; + + unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); + hasBOM_ = false; + if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } + + // RFC 4627: Section 3 + // "Since the first two characters of a JSON text will always be ASCII + // characters [RFC0020], it is possible to determine whether an octet + // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking + // at the pattern of nulls in the first four octets." + // 00 00 00 xx UTF-32BE + // 00 xx 00 xx UTF-16BE + // xx 00 00 00 UTF-32LE + // xx 00 xx 00 UTF-16LE + // xx xx xx xx UTF-8 + + if (!hasBOM_) { + unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + switch (pattern) { + case 0x08: type_ = kUTF32BE; break; + case 0x0A: type_ = kUTF16BE; break; + case 0x01: type_ = kUTF32LE; break; + case 0x05: type_ = kUTF16LE; break; + case 0x0F: type_ = kUTF8; break; + default: break; // Use type defined by user. + } + } + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + } + + typedef Ch (*TakeFunc)(InputByteStream& is); + InputByteStream* is_; + UTFType type_; + Ch current_; + TakeFunc takeFunc_; + bool hasBOM_; +}; + +//! Output stream wrapper with dynamically bound encoding and automatic encoding detection. +/*! + \tparam CharType Type of character for writing. + \tparam OutputByteStream type of output byte stream to be wrapped. +*/ +template +class AutoUTFOutputStream { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +public: + typedef CharType Ch; + + //! Constructor. + /*! + \param os output stream to be wrapped. + \param type UTF encoding type. + \param putBOM Whether to write BOM at the beginning of the stream. + */ + AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { + RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + + static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; + putFunc_ = f[type_]; + + if (putBOM) + PutBOM(); + } + + UTFType GetType() const { return type_; } + + void Put(Ch c) { putFunc_(*os_, c); } + void Flush() { os_->Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + AutoUTFOutputStream(const AutoUTFOutputStream&); + AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); + + void PutBOM() { + typedef void (*PutBOMFunc)(OutputByteStream&); + static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; + f[type_](*os_); + } + + typedef void (*PutFunc)(OutputByteStream&, Ch); + + OutputByteStream* os_; + UTFType type_; + PutFunc putFunc_; +}; + +#undef RAPIDJSON_ENCODINGS_FUNC + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/encodings.h b/ext/librethinkdbxx/src/rapidjson/encodings.h new file mode 100644 index 00000000..baa7c2b1 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/encodings.h @@ -0,0 +1,716 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ENCODINGS_H_ +#define RAPIDJSON_ENCODINGS_H_ + +#include "rapidjson.h" + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data +RAPIDJSON_DIAG_OFF(4702) // unreachable code +#elif defined(__GNUC__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +RAPIDJSON_DIAG_OFF(overflow) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Encoding + +/*! \class rapidjson::Encoding + \brief Concept for encoding of Unicode characters. + +\code +concept Encoding { + typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. + + enum { supportUnicode = 1 }; // or 0 if not supporting unicode + + //! \brief Encode a Unicode codepoint to an output stream. + //! \param os Output stream. + //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. + template + static void Encode(OutputStream& os, unsigned codepoint); + + //! \brief Decode a Unicode codepoint from an input stream. + //! \param is Input stream. + //! \param codepoint Output of the unicode codepoint. + //! \return true if a valid codepoint can be decoded from the stream. + template + static bool Decode(InputStream& is, unsigned* codepoint); + + //! \brief Validate one Unicode codepoint from an encoded stream. + //! \param is Input stream to obtain codepoint. + //! \param os Output for copying one codepoint. + //! \return true if it is valid. + //! \note This function just validating and copying the codepoint without actually decode it. + template + static bool Validate(InputStream& is, OutputStream& os); + + // The following functions are deal with byte streams. + + //! Take a character from input byte stream, skip BOM if exist. + template + static CharType TakeBOM(InputByteStream& is); + + //! Take a character from input byte stream. + template + static Ch Take(InputByteStream& is); + + //! Put BOM to output byte stream. + template + static void PutBOM(OutputByteStream& os); + + //! Put a character to output byte stream. + template + static void Put(OutputByteStream& os, Ch c); +}; +\endcode +*/ + +/////////////////////////////////////////////////////////////////////////////// +// UTF8 + +//! UTF-8 encoding. +/*! http://en.wikipedia.org/wiki/UTF-8 + http://tools.ietf.org/html/rfc3629 + \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. + \note implements Encoding concept +*/ +template +struct UTF8 { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + os.Put(static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + if (codepoint <= 0x7F) + PutUnsafe(os, static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) { + PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) { + PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { +#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + typename InputStream::Ch c = is.Take(); + if (!(c & 0x80)) { + *codepoint = static_cast(c); + return true; + } + + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) { + *codepoint = 0; + } else { + *codepoint = (0xFF >> type) & static_cast(c); + } + bool result = true; + switch (type) { + case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } +#undef COPY +#undef TRANS +#undef TAIL + } + + template + static bool Validate(InputStream& is, OutputStream& os) { +#define COPY() os.Put(c = is.Take()) +#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define TAIL() COPY(); TRANS(0x70) + Ch c; + COPY(); + if (!(c & 0x80)) + return true; + + bool result = true; + switch (GetRange(static_cast(c))) { + case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } +#undef COPY +#undef TRANS +#undef TAIL + } + + static unsigned char GetRange(unsigned char c) { + // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; + } + + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + typename InputByteStream::Ch c = Take(is); + if (static_cast(c) != 0xEFu) return c; + c = is.Take(); + if (static_cast(c) != 0xBBu) return c; + c = is.Take(); + if (static_cast(c) != 0xBFu) return c; + c = is.Take(); + return c; + } + + template + static Ch Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xEFu)); + os.Put(static_cast(0xBBu)); + os.Put(static_cast(0xBFu)); + } + + template + static void Put(OutputByteStream& os, Ch c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF16 + +//! UTF-16 encoding. +/*! http://en.wikipedia.org/wiki/UTF-16 + http://tools.ietf.org/html/rfc2781 + \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF16LE and UTF16BE, which handle endianness. +*/ +template +struct UTF16 { + typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + os.Put(static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + os.Put(static_cast((v >> 10) | 0xD800)); + os.Put((v & 0x3FF) | 0xDC00); + } + } + + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) { + RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + PutUnsafe(os, static_cast(codepoint)); + } + else { + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + PutUnsafe(os, static_cast((v >> 10) | 0xD800)); + PutUnsafe(os, (v & 0x3FF) | 0xDC00); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + typename InputStream::Ch c = is.Take(); + if (c < 0xD800 || c > 0xDFFF) { + *codepoint = static_cast(c); + return true; + } + else if (c <= 0xDBFF) { + *codepoint = (static_cast(c) & 0x3FF) << 10; + c = is.Take(); + *codepoint |= (static_cast(c) & 0x3FF); + *codepoint += 0x10000; + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + typename InputStream::Ch c; + os.Put(static_cast(c = is.Take())); + if (c < 0xD800 || c > 0xDFFF) + return true; + else if (c <= 0xDBFF) { + os.Put(c = is.Take()); + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } +}; + +//! UTF-16 little endian encoding. +template +struct UTF16LE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + } +}; + +//! UTF-16 big endian encoding. +template +struct UTF16BE : UTF16 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 8; + c |= static_cast(is.Take()); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(static_cast(static_cast(c) & 0xFFu)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// UTF32 + +//! UTF-32 encoding. +/*! http://en.wikipedia.org/wiki/UTF-32 + \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. + \note implements Encoding concept + + \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. + For streaming, use UTF32LE and UTF32BE, which handle endianness. +*/ +template +struct UTF32 { + typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(codepoint); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, codepoint); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c = is.Take(); + *codepoint = c; + return c <= 0x10FFFF; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c; + os.Put(c = is.Take()); + return c <= 0x10FFFF; + } +}; + +//! UTF-32 little endian enocoding. +template +struct UTF32LE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 24; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 24) & 0xFFu)); + } +}; + +//! UTF-32 big endian encoding. +template +struct UTF32BE : UTF32 { + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 24; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast(c & 0xFFu)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// ASCII + +//! ASCII encoding. +/*! http://en.wikipedia.org/wiki/ASCII + \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. + \note implements Encoding concept +*/ +template +struct ASCII { + typedef CharType Ch; + + enum { supportUnicode = 0 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + os.Put(static_cast(codepoint & 0xFF)); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + RAPIDJSON_ASSERT(codepoint <= 0x7F); + PutUnsafe(os, static_cast(codepoint & 0xFF)); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) { + uint8_t c = static_cast(is.Take()); + *codepoint = c; + return c <= 0X7F; + } + + template + static bool Validate(InputStream& is, OutputStream& os) { + uint8_t c = static_cast(is.Take()); + os.Put(static_cast(c)); + return c <= 0x7F; + } + + template + static CharType TakeBOM(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + uint8_t c = static_cast(Take(is)); + return static_cast(c); + } + + template + static Ch Take(InputByteStream& is) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + (void)os; + } + + template + static void Put(OutputByteStream& os, Ch c) { + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// AutoUTF + +//! Runtime-specified UTF encoding type of a stream. +enum UTFType { + kUTF8 = 0, //!< UTF-8. + kUTF16LE = 1, //!< UTF-16 little endian. + kUTF16BE = 2, //!< UTF-16 big endian. + kUTF32LE = 3, //!< UTF-32 little endian. + kUTF32BE = 4 //!< UTF-32 big endian. +}; + +//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. +/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). +*/ +template +struct AutoUTF { + typedef CharType Ch; + + enum { supportUnicode = 1 }; + +#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x + + template + RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { + typedef bool (*DecodeFunc)(InputStream&, unsigned*); + static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; + return (*f[is.GetType()])(is, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + typedef bool (*ValidateFunc)(InputStream&, OutputStream&); + static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; + return (*f[is.GetType()])(is, os); + } + +#undef RAPIDJSON_ENCODINGS_FUNC +}; + +/////////////////////////////////////////////////////////////////////////////// +// Transcoder + +//! Encoding conversion. +template +struct Transcoder { + //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::Encode(os, codepoint); + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::EncodeUnsafe(os, codepoint); + return true; + } + + //! Validate one Unicode codepoint from an encoded stream. + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + return Transcode(is, os); // Since source/target encoding is different, must transcode. + } +}; + +// Forward declaration. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c); + +//! Specialization of Transcoder with same source and target encoding. +template +struct Transcoder { + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + return Encoding::Validate(is, os); // source/target encoding are the same + } +}; + +RAPIDJSON_NAMESPACE_END + +#if defined(__GNUC__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ENCODINGS_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/error/en.h b/ext/librethinkdbxx/src/rapidjson/error/en.h new file mode 100644 index 00000000..2db838bf --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/error/en.h @@ -0,0 +1,74 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ERROR_EN_H_ +#define RAPIDJSON_ERROR_EN_H_ + +#include "error.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(covered-switch-default) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Maps error code of parsing into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param parseErrorCode Error code obtained in parsing. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { + switch (parseErrorCode) { + case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); + case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); + + case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); + + case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); + case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); + case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); + + case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); + + case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); + case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); + case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); + case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); + case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); + + case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); + case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); + case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); + + case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); + case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_EN_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/error/error.h b/ext/librethinkdbxx/src/rapidjson/error/error.h new file mode 100644 index 00000000..95cb31a7 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/error/error.h @@ -0,0 +1,155 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ERROR_ERROR_H_ +#define RAPIDJSON_ERROR_ERROR_H_ + +#include "../rapidjson.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +/*! \file error.h */ + +/*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ERROR_CHARTYPE + +//! Character type of error messages. +/*! \ingroup RAPIDJSON_ERRORS + The default character type is \c char. + On Windows, user can define this macro as \c TCHAR for supporting both + unicode/non-unicode settings. +*/ +#ifndef RAPIDJSON_ERROR_CHARTYPE +#define RAPIDJSON_ERROR_CHARTYPE char +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ERROR_STRING + +//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. +/*! \ingroup RAPIDJSON_ERRORS + By default this conversion macro does nothing. + On Windows, user can define this macro as \c _T(x) for supporting both + unicode/non-unicode settings. +*/ +#ifndef RAPIDJSON_ERROR_STRING +#define RAPIDJSON_ERROR_STRING(x) x +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// ParseErrorCode + +//! Error code of parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericReader::Parse, GenericReader::GetParseErrorCode +*/ +enum ParseErrorCode { + kParseErrorNone = 0, //!< No error. + + kParseErrorDocumentEmpty, //!< The document is empty. + kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. + + kParseErrorValueInvalid, //!< Invalid value. + + kParseErrorObjectMissName, //!< Missing a name for object member. + kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. + kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. + + kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. + + kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. + kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. + kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. + kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. + kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. + + kParseErrorNumberTooBig, //!< Number too big to be stored in double. + kParseErrorNumberMissFraction, //!< Miss fraction part in number. + kParseErrorNumberMissExponent, //!< Miss exponent in number. + + kParseErrorTermination, //!< Parsing was terminated. + kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. +}; + +//! Result of parsing (wraps ParseErrorCode) +/*! + \ingroup RAPIDJSON_ERRORS + \code + Document doc; + ParseResult ok = doc.Parse("[42]"); + if (!ok) { + fprintf(stderr, "JSON parse error: %s (%u)", + GetParseError_En(ok.Code()), ok.Offset()); + exit(EXIT_FAILURE); + } + \endcode + \see GenericReader::Parse, GenericDocument::Parse +*/ +struct ParseResult { +public: + //! Default constructor, no error. + ParseResult() : code_(kParseErrorNone), offset_(0) {} + //! Constructor to set an error. + ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} + + //! Get the error code. + ParseErrorCode Code() const { return code_; } + //! Get the error offset, if \ref IsError(), 0 otherwise. + size_t Offset() const { return offset_; } + + //! Conversion to \c bool, returns \c true, iff !\ref IsError(). + operator bool() const { return !IsError(); } + //! Whether the result is an error. + bool IsError() const { return code_ != kParseErrorNone; } + + bool operator==(const ParseResult& that) const { return code_ == that.code_; } + bool operator==(ParseErrorCode code) const { return code_ == code; } + friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + + //! Reset error code. + void Clear() { Set(kParseErrorNone); } + //! Update error code and offset. + void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } + +private: + ParseErrorCode code_; + size_t offset_; +}; + +//! Function pointer type of GetParseError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetParseError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetParseErrorFunc GetParseError = GetParseError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_ERROR_ERROR_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/filereadstream.h b/ext/librethinkdbxx/src/rapidjson/filereadstream.h new file mode 100644 index 00000000..b56ea13b --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/filereadstream.h @@ -0,0 +1,99 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FILEREADSTREAM_H_ +#define RAPIDJSON_FILEREADSTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! File byte stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileReadStream { +public: + typedef char Ch; //!< Character type (byte). + + //! Constructor. + /*! + \param fp File pointer opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(fp_ != 0); + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } + + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return (current_ + 4 <= bufferLast_) ? current_ : 0; + } + +private: + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (readCount_ < bufferSize_) { + buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } + } + } + + std::FILE* fp_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/filewritestream.h b/ext/librethinkdbxx/src/rapidjson/filewritestream.h new file mode 100644 index 00000000..6378dd60 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/filewritestream.h @@ -0,0 +1,104 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FILEWRITESTREAM_H_ +#define RAPIDJSON_FILEWRITESTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of C file stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileWriteStream { +public: + typedef char Ch; //!< Character type. Only support char. + + FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { + RAPIDJSON_ASSERT(fp_ != 0); + } + + void Put(char c) { + if (current_ >= bufferEnd_) + Flush(); + + *current_++ = c; + } + + void PutN(char c, size_t n) { + size_t avail = static_cast(bufferEnd_ - current_); + while (n > avail) { + std::memset(current_, c, avail); + current_ += avail; + Flush(); + n -= avail; + avail = static_cast(bufferEnd_ - current_); + } + + if (n > 0) { + std::memset(current_, c, n); + current_ += n; + } + } + + void Flush() { + if (current_ != buffer_) { + size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } + current_ = buffer_; + } + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + // Prohibit copy constructor & assignment operator. + FileWriteStream(const FileWriteStream&); + FileWriteStream& operator=(const FileWriteStream&); + + std::FILE* fp_; + char *buffer_; + char *bufferEnd_; + char *current_; +}; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(FileWriteStream& stream, char c, size_t n) { + stream.PutN(c, n); +} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/fwd.h b/ext/librethinkdbxx/src/rapidjson/fwd.h new file mode 100644 index 00000000..e8104e84 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/fwd.h @@ -0,0 +1,151 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FWD_H_ +#define RAPIDJSON_FWD_H_ + +#include "rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN + +// encodings.h + +template struct UTF8; +template struct UTF16; +template struct UTF16BE; +template struct UTF16LE; +template struct UTF32; +template struct UTF32BE; +template struct UTF32LE; +template struct ASCII; +template struct AutoUTF; + +template +struct Transcoder; + +// allocators.h + +class CrtAllocator; + +template +class MemoryPoolAllocator; + +// stream.h + +template +struct GenericStringStream; + +typedef GenericStringStream > StringStream; + +template +struct GenericInsituStringStream; + +typedef GenericInsituStringStream > InsituStringStream; + +// stringbuffer.h + +template +class GenericStringBuffer; + +typedef GenericStringBuffer, CrtAllocator> StringBuffer; + +// filereadstream.h + +class FileReadStream; + +// filewritestream.h + +class FileWriteStream; + +// memorybuffer.h + +template +struct GenericMemoryBuffer; + +typedef GenericMemoryBuffer MemoryBuffer; + +// memorystream.h + +struct MemoryStream; + +// reader.h + +template +struct BaseReaderHandler; + +template +class GenericReader; + +typedef GenericReader, UTF8, CrtAllocator> Reader; + +// writer.h + +template +class Writer; + +// prettywriter.h + +template +class PrettyWriter; + +// document.h + +template +struct GenericMember; + +template +class GenericMemberIterator; + +template +struct GenericStringRef; + +template +class GenericValue; + +typedef GenericValue, MemoryPoolAllocator > Value; + +template +class GenericDocument; + +typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; + +// pointer.h + +template +class GenericPointer; + +typedef GenericPointer Pointer; + +// schema.h + +template +class IGenericRemoteSchemaDocumentProvider; + +template +class GenericSchemaDocument; + +typedef GenericSchemaDocument SchemaDocument; +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +template < + typename SchemaDocumentType, + typename OutputHandler, + typename StateAllocator> +class GenericSchemaValidator; + +typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_RAPIDJSONFWD_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h b/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h new file mode 100644 index 00000000..9d3e88c9 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/biginteger.h @@ -0,0 +1,290 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_BIGINTEGER_H_ +#define RAPIDJSON_BIGINTEGER_H_ + +#include "../rapidjson.h" + +#if defined(_MSC_VER) && defined(_M_AMD64) +#include // for _umul128 +#pragma intrinsic(_umul128) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +class BigInteger { +public: + typedef uint64_t Type; + + BigInteger(const BigInteger& rhs) : count_(rhs.count_) { + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + + explicit BigInteger(uint64_t u) : count_(1) { + digits_[0] = u; + } + + BigInteger(const char* decimals, size_t length) : count_(1) { + RAPIDJSON_ASSERT(length > 0); + digits_[0] = 0; + size_t i = 0; + const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 + while (length >= kMaxDigitPerIteration) { + AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); + length -= kMaxDigitPerIteration; + i += kMaxDigitPerIteration; + } + + if (length > 0) + AppendDecimal64(decimals + i, decimals + i + length); + } + + BigInteger& operator=(const BigInteger &rhs) + { + if (this != &rhs) { + count_ = rhs.count_; + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + return *this; + } + + BigInteger& operator=(uint64_t u) { + digits_[0] = u; + count_ = 1; + return *this; + } + + BigInteger& operator+=(uint64_t u) { + Type backup = digits_[0]; + digits_[0] += u; + for (size_t i = 0; i < count_ - 1; i++) { + if (digits_[i] >= backup) + return *this; // no carry + backup = digits_[i + 1]; + digits_[i + 1] += 1; + } + + // Last carry + if (digits_[count_ - 1] < backup) + PushBack(1); + + return *this; + } + + BigInteger& operator*=(uint64_t u) { + if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) { + uint64_t hi; + digits_[i] = MulAdd64(digits_[i], u, k, &hi); + k = hi; + } + + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator*=(uint32_t u) { + if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) { + const uint64_t c = digits_[i] >> 32; + const uint64_t d = digits_[i] & 0xFFFFFFFF; + const uint64_t uc = u * c; + const uint64_t ud = u * d; + const uint64_t p0 = ud + k; + const uint64_t p1 = uc + (p0 >> 32); + digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); + k = p1 >> 32; + } + + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator<<=(size_t shift) { + if (IsZero() || shift == 0) return *this; + + size_t offset = shift / kTypeBit; + size_t interShift = shift % kTypeBit; + RAPIDJSON_ASSERT(count_ + offset <= kCapacity); + + if (interShift == 0) { + std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); + count_ += offset; + } + else { + digits_[count_] = 0; + for (size_t i = count_; i > 0; i--) + digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); + digits_[offset] = digits_[0] << interShift; + count_ += offset; + if (digits_[count_]) + count_++; + } + + std::memset(digits_, 0, offset * sizeof(Type)); + + return *this; + } + + bool operator==(const BigInteger& rhs) const { + return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; + } + + bool operator==(const Type rhs) const { + return count_ == 1 && digits_[0] == rhs; + } + + BigInteger& MultiplyPow5(unsigned exp) { + static const uint32_t kPow5[12] = { + 5, + 5 * 5, + 5 * 5 * 5, + 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 + }; + if (exp == 0) return *this; + for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 + for (; exp >= 13; exp -= 13) *this *= static_cast(1220703125u); // 5^13 + if (exp > 0) *this *= kPow5[exp - 1]; + return *this; + } + + // Compute absolute difference of this and rhs. + // Assume this != rhs + bool Difference(const BigInteger& rhs, BigInteger* out) const { + int cmp = Compare(rhs); + RAPIDJSON_ASSERT(cmp != 0); + const BigInteger *a, *b; // Makes a > b + bool ret; + if (cmp < 0) { a = &rhs; b = this; ret = true; } + else { a = this; b = &rhs; ret = false; } + + Type borrow = 0; + for (size_t i = 0; i < a->count_; i++) { + Type d = a->digits_[i] - borrow; + if (i < b->count_) + d -= b->digits_[i]; + borrow = (d > a->digits_[i]) ? 1 : 0; + out->digits_[i] = d; + if (d != 0) + out->count_ = i + 1; + } + + return ret; + } + + int Compare(const BigInteger& rhs) const { + if (count_ != rhs.count_) + return count_ < rhs.count_ ? -1 : 1; + + for (size_t i = count_; i-- > 0;) + if (digits_[i] != rhs.digits_[i]) + return digits_[i] < rhs.digits_[i] ? -1 : 1; + + return 0; + } + + size_t GetCount() const { return count_; } + Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; } + bool IsZero() const { return count_ == 1 && digits_[0] == 0; } + +private: + void AppendDecimal64(const char* begin, const char* end) { + uint64_t u = ParseUint64(begin, end); + if (IsZero()) + *this = u; + else { + unsigned exp = static_cast(end - begin); + (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u + } + } + + void PushBack(Type digit) { + RAPIDJSON_ASSERT(count_ < kCapacity); + digits_[count_++] = digit; + } + + static uint64_t ParseUint64(const char* begin, const char* end) { + uint64_t r = 0; + for (const char* p = begin; p != end; ++p) { + RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); + r = r * 10u + static_cast(*p - '0'); + } + return r; + } + + // Assume a * b + k < 2^128 + static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t low = _umul128(a, b, outHigh) + k; + if (low < k) + (*outHigh)++; + return low; +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(a) * static_cast(b); + p += k; + *outHigh = static_cast(p >> 64); + return static_cast(p); +#else + const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; + uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; + x1 += (x0 >> 32); // can't give carry + x1 += x2; + if (x1 < x2) + x3 += (static_cast(1) << 32); + uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); + uint64_t hi = x3 + (x1 >> 32); + + lo += k; + if (lo < k) + hi++; + *outHigh = hi; + return lo; +#endif + } + + static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 + static const size_t kCapacity = kBitCount / sizeof(Type); + static const size_t kTypeBit = sizeof(Type) * 8; + + Type digits_[kCapacity]; + size_t count_; +}; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_BIGINTEGER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h b/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h new file mode 100644 index 00000000..c9fefdc6 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/diyfp.h @@ -0,0 +1,258 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef RAPIDJSON_DIYFP_H_ +#define RAPIDJSON_DIYFP_H_ + +#include "../rapidjson.h" + +#if defined(_MSC_VER) && defined(_M_AMD64) +#include +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_umul128) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +struct DiyFp { + DiyFp() : f(), e() {} + + DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} + + explicit DiyFp(double d) { + union { + double d; + uint64_t u64; + } u = { d }; + + int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); + uint64_t significand = (u.u64 & kDpSignificandMask); + if (biased_e != 0) { + f = significand + kDpHiddenBit; + e = biased_e - kDpExponentBias; + } + else { + f = significand; + e = kDpMinExponent + 1; + } + } + + DiyFp operator-(const DiyFp& rhs) const { + return DiyFp(f - rhs.f, e); + } + + DiyFp operator*(const DiyFp& rhs) const { +#if defined(_MSC_VER) && defined(_M_AMD64) + uint64_t h; + uint64_t l = _umul128(f, rhs.f, &h); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(f) * static_cast(rhs.f); + uint64_t h = static_cast(p >> 64); + uint64_t l = static_cast(p); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); +#else + const uint64_t M32 = 0xFFFFFFFF; + const uint64_t a = f >> 32; + const uint64_t b = f & M32; + const uint64_t c = rhs.f >> 32; + const uint64_t d = rhs.f & M32; + const uint64_t ac = a * c; + const uint64_t bc = b * c; + const uint64_t ad = a * d; + const uint64_t bd = b * d; + uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); + tmp += 1U << 31; /// mult_round + return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); +#endif + } + + DiyFp Normalize() const { +#if defined(_MSC_VER) && defined(_M_AMD64) + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp(f << (63 - index), e - (63 - index)); +#elif defined(__GNUC__) && __GNUC__ >= 4 + int s = __builtin_clzll(f); + return DiyFp(f << s, e - s); +#else + DiyFp res = *this; + while (!(res.f & (static_cast(1) << 63))) { + res.f <<= 1; + res.e--; + } + return res; +#endif + } + + DiyFp NormalizeBoundary() const { + DiyFp res = *this; + while (!(res.f & (kDpHiddenBit << 1))) { + res.f <<= 1; + res.e--; + } + res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); + res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); + return res; + } + + void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { + DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); + DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); + mi.f <<= mi.e - pl.e; + mi.e = pl.e; + *plus = pl; + *minus = mi; + } + + double ToDouble() const { + union { + double d; + uint64_t u64; + }u; + const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : + static_cast(e + kDpExponentBias); + u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); + return u.d; + } + + static const int kDiySignificandSize = 64; + static const int kDpSignificandSize = 52; + static const int kDpExponentBias = 0x3FF + kDpSignificandSize; + static const int kDpMaxExponent = 0x7FF - kDpExponentBias; + static const int kDpMinExponent = -kDpExponentBias; + static const int kDpDenormalExponent = -kDpExponentBias + 1; + static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + uint64_t f; + int e; +}; + +inline DiyFp GetCachedPowerByIndex(size_t index) { + // 10^-348, 10^-340, ..., 10^340 + static const uint64_t kCachedPowers_F[] = { + RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), + RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), + RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), + RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), + RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), + RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), + RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), + RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), + RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), + RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), + RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), + RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), + RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), + RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), + RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), + RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), + RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), + RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), + RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), + RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), + RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), + RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), + RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), + RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), + RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), + RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), + RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), + RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), + RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), + RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), + RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), + RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), + RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), + RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), + RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), + RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), + RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), + RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), + RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), + RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), + RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), + RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), + RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), + RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) + }; + static const int16_t kCachedPowers_E[] = { + -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, + -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, + -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, + -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, + -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, + 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, + 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, + 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, + 907, 933, 960, 986, 1013, 1039, 1066 + }; + return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); +} + +inline DiyFp GetCachedPower(int e, int* K) { + + //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; + double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive + int k = static_cast(dk); + if (dk - k > 0.0) + k++; + + unsigned index = static_cast((k >> 3) + 1); + *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table + + return GetCachedPowerByIndex(index); +} + +inline DiyFp GetCachedPower10(int exp, int *outExp) { + unsigned index = (static_cast(exp) + 348u) / 8u; + *outExp = -348 + static_cast(index) * 8; + return GetCachedPowerByIndex(index); + } + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +RAPIDJSON_DIAG_OFF(padded) +#endif + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_DIYFP_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h b/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h new file mode 100644 index 00000000..8d6350e6 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/dtoa.h @@ -0,0 +1,245 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// This is a C++ header-only implementation of Grisu2 algorithm from the publication: +// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with +// integers." ACM Sigplan Notices 45.6 (2010): 233-243. + +#ifndef RAPIDJSON_DTOA_ +#define RAPIDJSON_DTOA_ + +#include "itoa.h" // GetDigitsLut() +#include "diyfp.h" +#include "ieee754.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 +#endif + +inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { + while (rest < wp_w && delta - rest >= ten_kappa && + (rest + ten_kappa < wp_w || /// closer + wp_w - rest > rest + ten_kappa - wp_w)) { + buffer[len - 1]--; + rest += ten_kappa; + } +} + +inline unsigned CountDecimalDigit32(uint32_t n) { + // Simple pure C++ implementation was faster than __builtin_clz version in this situation. + if (n < 10) return 1; + if (n < 100) return 2; + if (n < 1000) return 3; + if (n < 10000) return 4; + if (n < 100000) return 5; + if (n < 1000000) return 6; + if (n < 10000000) return 7; + if (n < 100000000) return 8; + // Will not reach 10 digits in DigitGen() + //if (n < 1000000000) return 9; + //return 10; + return 9; +} + +inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { + static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); + const DiyFp wp_w = Mp - W; + uint32_t p1 = static_cast(Mp.f >> -one.e); + uint64_t p2 = Mp.f & (one.f - 1); + unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] + *len = 0; + + while (kappa > 0) { + uint32_t d = 0; + switch (kappa) { + case 9: d = p1 / 100000000; p1 %= 100000000; break; + case 8: d = p1 / 10000000; p1 %= 10000000; break; + case 7: d = p1 / 1000000; p1 %= 1000000; break; + case 6: d = p1 / 100000; p1 %= 100000; break; + case 5: d = p1 / 10000; p1 %= 10000; break; + case 4: d = p1 / 1000; p1 %= 1000; break; + case 3: d = p1 / 100; p1 %= 100; break; + case 2: d = p1 / 10; p1 %= 10; break; + case 1: d = p1; p1 = 0; break; + default:; + } + if (d || *len) + buffer[(*len)++] = static_cast('0' + static_cast(d)); + kappa--; + uint64_t tmp = (static_cast(p1) << -one.e) + p2; + if (tmp <= delta) { + *K += kappa; + GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + return; + } + } + + // kappa = 0 + for (;;) { + p2 *= 10; + delta *= 10; + char d = static_cast(p2 >> -one.e); + if (d || *len) + buffer[(*len)++] = static_cast('0' + d); + p2 &= one.f - 1; + kappa--; + if (p2 < delta) { + *K += kappa; + int index = -static_cast(kappa); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast(kappa)] : 0)); + return; + } + } +} + +inline void Grisu2(double value, char* buffer, int* length, int* K) { + const DiyFp v(value); + DiyFp w_m, w_p; + v.NormalizedBoundaries(&w_m, &w_p); + + const DiyFp c_mk = GetCachedPower(w_p.e, K); + const DiyFp W = v.Normalize() * c_mk; + DiyFp Wp = w_p * c_mk; + DiyFp Wm = w_m * c_mk; + Wm.f++; + Wp.f--; + DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); +} + +inline char* WriteExponent(int K, char* buffer) { + if (K < 0) { + *buffer++ = '-'; + K = -K; + } + + if (K >= 100) { + *buffer++ = static_cast('0' + static_cast(K / 100)); + K %= 100; + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else if (K >= 10) { + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else + *buffer++ = static_cast('0' + static_cast(K)); + + return buffer; +} + +inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { + const int kk = length + k; // 10^(kk-1) <= v < 10^kk + + if (0 <= k && kk <= 21) { + // 1234e7 -> 12340000000 + for (int i = length; i < kk; i++) + buffer[i] = '0'; + buffer[kk] = '.'; + buffer[kk + 1] = '0'; + return &buffer[kk + 2]; + } + else if (0 < kk && kk <= 21) { + // 1234e-2 -> 12.34 + std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); + buffer[kk] = '.'; + if (0 > k + maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[kk + 2]; // Reserve one zero + } + else + return &buffer[length + 1]; + } + else if (-6 < kk && kk <= 0) { + // 1234e-6 -> 0.001234 + const int offset = 2 - kk; + std::memmove(&buffer[offset], &buffer[0], static_cast(length)); + buffer[0] = '0'; + buffer[1] = '.'; + for (int i = 2; i < offset; i++) + buffer[i] = '0'; + if (length - kk > maxDecimalPlaces) { + // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = maxDecimalPlaces + 1; i > 2; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[3]; // Reserve one zero + } + else + return &buffer[length + offset]; + } + else if (kk < -maxDecimalPlaces) { + // Truncate to zero + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else if (length == 1) { + // 1e30 + buffer[1] = 'e'; + return WriteExponent(kk - 1, &buffer[2]); + } + else { + // 1234e30 -> 1.234e33 + std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); + buffer[1] = '.'; + buffer[length + 1] = 'e'; + return WriteExponent(kk - 1, &buffer[0 + length + 2]); + } +} + +inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { + RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); + Double d(value); + if (d.IsZero()) { + if (d.Sign()) + *buffer++ = '-'; // -0.0, Issue #289 + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else { + if (value < 0) { + *buffer++ = '-'; + value = -value; + } + int length, K; + Grisu2(value, buffer, &length, &K); + return Prettify(buffer, length, K, maxDecimalPlaces); + } +} + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_DTOA_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h b/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h new file mode 100644 index 00000000..82bb0b99 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/ieee754.h @@ -0,0 +1,78 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_IEEE754_ +#define RAPIDJSON_IEEE754_ + +#include "../rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +class Double { +public: + Double() {} + Double(double d) : d_(d) {} + Double(uint64_t u) : u_(u) {} + + double Value() const { return d_; } + uint64_t Uint64Value() const { return u_; } + + double NextPositiveDouble() const { + RAPIDJSON_ASSERT(!Sign()); + return Double(u_ + 1).Value(); + } + + bool Sign() const { return (u_ & kSignMask) != 0; } + uint64_t Significand() const { return u_ & kSignificandMask; } + int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } + + bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } + bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } + bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } + bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } + bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } + + uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } + int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } + uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } + + static unsigned EffectiveSignificandSize(int order) { + if (order >= -1021) + return 53; + else if (order <= -1074) + return 0; + else + return static_cast(order) + 1074; + } + +private: + static const int kSignificandSize = 52; + static const int kExponentBias = 0x3FF; + static const int kDenormalExponent = 1 - kExponentBias; + static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); + static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + union { + double d_; + uint64_t u_; + }; +}; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_IEEE754_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/itoa.h b/ext/librethinkdbxx/src/rapidjson/internal/itoa.h new file mode 100644 index 00000000..01a4e7e7 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/itoa.h @@ -0,0 +1,304 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ITOA_ +#define RAPIDJSON_ITOA_ + +#include "../rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline const char* GetDigitsLut() { + static const char cDigitsLut[200] = { + '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' + }; + return cDigitsLut; +} + +inline char* u32toa(uint32_t value, char* buffer) { + const char* cDigitsLut = GetDigitsLut(); + + if (value < 10000) { + const uint32_t d1 = (value / 100) << 1; + const uint32_t d2 = (value % 100) << 1; + + if (value >= 1000) + *buffer++ = cDigitsLut[d1]; + if (value >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else if (value < 100000000) { + // value = bbbbcccc + const uint32_t b = value / 10000; + const uint32_t c = value % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + else { + // value = aabbbbcccc in decimal + + const uint32_t a = value / 100000000; // 1 to 42 + value %= 100000000; + + if (a >= 10) { + const unsigned i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else + *buffer++ = static_cast('0' + static_cast(a)); + + const uint32_t b = value / 10000; // 0 to 9999 + const uint32_t c = value % 10000; // 0 to 9999 + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + return buffer; +} + +inline char* i32toa(int32_t value, char* buffer) { + uint32_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + + return u32toa(u, buffer); +} + +inline char* u64toa(uint64_t value, char* buffer) { + const char* cDigitsLut = GetDigitsLut(); + const uint64_t kTen8 = 100000000; + const uint64_t kTen9 = kTen8 * 10; + const uint64_t kTen10 = kTen8 * 100; + const uint64_t kTen11 = kTen8 * 1000; + const uint64_t kTen12 = kTen8 * 10000; + const uint64_t kTen13 = kTen8 * 100000; + const uint64_t kTen14 = kTen8 * 1000000; + const uint64_t kTen15 = kTen8 * 10000000; + const uint64_t kTen16 = kTen8 * kTen8; + + if (value < kTen8) { + uint32_t v = static_cast(value); + if (v < 10000) { + const uint32_t d1 = (v / 100) << 1; + const uint32_t d2 = (v % 100) << 1; + + if (v >= 1000) + *buffer++ = cDigitsLut[d1]; + if (v >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (v >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else { + // value = bbbbcccc + const uint32_t b = v / 10000; + const uint32_t c = v % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + } + else if (value < kTen16) { + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + if (value >= kTen15) + *buffer++ = cDigitsLut[d1]; + if (value >= kTen14) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= kTen13) + *buffer++ = cDigitsLut[d2]; + if (value >= kTen12) + *buffer++ = cDigitsLut[d2 + 1]; + if (value >= kTen11) + *buffer++ = cDigitsLut[d3]; + if (value >= kTen10) + *buffer++ = cDigitsLut[d3 + 1]; + if (value >= kTen9) + *buffer++ = cDigitsLut[d4]; + if (value >= kTen8) + *buffer++ = cDigitsLut[d4 + 1]; + + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + else { + const uint32_t a = static_cast(value / kTen16); // 1 to 1844 + value %= kTen16; + + if (a < 10) + *buffer++ = static_cast('0' + static_cast(a)); + else if (a < 100) { + const uint32_t i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else if (a < 1000) { + *buffer++ = static_cast('0' + static_cast(a / 100)); + + const uint32_t i = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else { + const uint32_t i = (a / 100) << 1; + const uint32_t j = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + *buffer++ = cDigitsLut[j]; + *buffer++ = cDigitsLut[j + 1]; + } + + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + + return buffer; +} + +inline char* i64toa(int64_t value, char* buffer) { + uint64_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + + return u64toa(u, buffer); +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ITOA_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/meta.h b/ext/librethinkdbxx/src/rapidjson/internal/meta.h new file mode 100644 index 00000000..5a9aaa42 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/meta.h @@ -0,0 +1,181 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_META_H_ +#define RAPIDJSON_INTERNAL_META_H_ + +#include "../rapidjson.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif +#if defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(6334) +#endif + +#if RAPIDJSON_HAS_CXX11_TYPETRAITS +#include +#endif + +//@cond RAPIDJSON_INTERNAL +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +// Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching +template struct Void { typedef void Type; }; + +/////////////////////////////////////////////////////////////////////////////// +// BoolType, TrueType, FalseType +// +template struct BoolType { + static const bool Value = Cond; + typedef BoolType Type; +}; +typedef BoolType TrueType; +typedef BoolType FalseType; + + +/////////////////////////////////////////////////////////////////////////////// +// SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr +// + +template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; +template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; +template struct SelectIfCond : SelectIfImpl::template Apply {}; +template struct SelectIf : SelectIfCond {}; + +template struct AndExprCond : FalseType {}; +template <> struct AndExprCond : TrueType {}; +template struct OrExprCond : TrueType {}; +template <> struct OrExprCond : FalseType {}; + +template struct BoolExpr : SelectIf::Type {}; +template struct NotExpr : SelectIf::Type {}; +template struct AndExpr : AndExprCond::Type {}; +template struct OrExpr : OrExprCond::Type {}; + + +/////////////////////////////////////////////////////////////////////////////// +// AddConst, MaybeAddConst, RemoveConst +template struct AddConst { typedef const T Type; }; +template struct MaybeAddConst : SelectIfCond {}; +template struct RemoveConst { typedef T Type; }; +template struct RemoveConst { typedef T Type; }; + + +/////////////////////////////////////////////////////////////////////////////// +// IsSame, IsConst, IsMoreConst, IsPointer +// +template struct IsSame : FalseType {}; +template struct IsSame : TrueType {}; + +template struct IsConst : FalseType {}; +template struct IsConst : TrueType {}; + +template +struct IsMoreConst + : AndExpr::Type, typename RemoveConst::Type>, + BoolType::Value >= IsConst::Value> >::Type {}; + +template struct IsPointer : FalseType {}; +template struct IsPointer : TrueType {}; + +/////////////////////////////////////////////////////////////////////////////// +// IsBaseOf +// +#if RAPIDJSON_HAS_CXX11_TYPETRAITS + +template struct IsBaseOf + : BoolType< ::std::is_base_of::value> {}; + +#else // simplified version adopted from Boost + +template struct IsBaseOfImpl { + RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); + RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); + + typedef char (&Yes)[1]; + typedef char (&No) [2]; + + template + static Yes Check(const D*, T); + static No Check(const B*, int); + + struct Host { + operator const B*() const; + operator const D*(); + }; + + enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; +}; + +template struct IsBaseOf + : OrExpr, BoolExpr > >::Type {}; + +#endif // RAPIDJSON_HAS_CXX11_TYPETRAITS + + +////////////////////////////////////////////////////////////////////////// +// EnableIf / DisableIf +// +template struct EnableIfCond { typedef T Type; }; +template struct EnableIfCond { /* empty */ }; + +template struct DisableIfCond { typedef T Type; }; +template struct DisableIfCond { /* empty */ }; + +template +struct EnableIf : EnableIfCond {}; + +template +struct DisableIf : DisableIfCond {}; + +// SFINAE helpers +struct SfinaeTag {}; +template struct RemoveSfinaeTag; +template struct RemoveSfinaeTag { typedef T Type; }; + +#define RAPIDJSON_REMOVEFPTR_(type) \ + typename ::RAPIDJSON_NAMESPACE::internal::RemoveSfinaeTag \ + < ::RAPIDJSON_NAMESPACE::internal::SfinaeTag&(*) type>::Type + +#define RAPIDJSON_ENABLEIF(cond) \ + typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ + ::Type * = NULL + +#define RAPIDJSON_DISABLEIF(cond) \ + typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ + ::Type * = NULL + +#define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ + typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ + ::Type + +#define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ + typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ + ::Type + +} // namespace internal +RAPIDJSON_NAMESPACE_END +//@endcond + +#if defined(__GNUC__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_META_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/pow10.h b/ext/librethinkdbxx/src/rapidjson/internal/pow10.h new file mode 100644 index 00000000..02f475d7 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/pow10.h @@ -0,0 +1,55 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_POW10_ +#define RAPIDJSON_POW10_ + +#include "../rapidjson.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Computes integer powers of 10 in double (10.0^n). +/*! This function uses lookup table for fast and accurate results. + \param n non-negative exponent. Must <= 308. + \return 10.0^n +*/ +inline double Pow10(int n) { + static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes + 1e+0, + 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, + 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, + 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, + 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, + 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, + 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, + 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, + 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, + 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, + 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, + 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, + 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, + 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, + 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, + 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 + }; + RAPIDJSON_ASSERT(n >= 0 && n <= 308); + return e[n]; +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_POW10_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/regex.h b/ext/librethinkdbxx/src/rapidjson/internal/regex.h new file mode 100644 index 00000000..422a5240 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/regex.h @@ -0,0 +1,701 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_REGEX_H_ +#define RAPIDJSON_INTERNAL_REGEX_H_ + +#include "../allocators.h" +#include "../stream.h" +#include "stack.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +#ifndef RAPIDJSON_REGEX_VERBOSE +#define RAPIDJSON_REGEX_VERBOSE 0 +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// GenericRegex + +static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 +static const SizeType kRegexInvalidRange = ~SizeType(0); + +//! Regular expression engine with subset of ECMAscript grammar. +/*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html +*/ +template +class GenericRegex { +public: + typedef typename Encoding::Ch Ch; + + GenericRegex(const Ch* source, Allocator* allocator = 0) : + states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() + { + GenericStringStream ss(source); + DecodedStream > ds(ss); + Parse(ds); + } + + ~GenericRegex() { + Allocator::Free(stateSet_); + } + + bool IsValid() const { + return root_ != kRegexInvalidState; + } + + template + bool Match(InputStream& is) const { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) const { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) const { + return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); + } + + bool Search(const Ch* s) const { + GenericStringStream is(s); + return Search(is); + } + +private: + enum Operator { + kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis + }; + + static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' + static const unsigned kRangeCharacterClass = 0xFFFFFFFE; + static const unsigned kRangeNegationFlag = 0x80000000; + + struct Range { + unsigned start; // + unsigned end; + SizeType next; + }; + + struct State { + SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; + }; + + struct Frag { + Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; + }; + + template + class DecodedStream { + public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + + private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; + }; + + State& GetState(SizeType index) { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + const State& GetState(SizeType index) const { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + Range& GetRange(SizeType index) { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + const Range& GetRange(SizeType index) const { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + template + void Parse(DecodedStream& ds) { + Allocator allocator; + Stack operandStack(&allocator, 256); // Frag + Stack operatorStack(&allocator, 256); // Operator + Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) { + switch (codepoint = ds.Take()) { + case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { + unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') { + ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; + } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { + SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); + } + } + + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) { + Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; + +#if RAPIDJSON_REGEX_VERBOSE + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) { + State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); + } + printf("\n"); +#endif + } + + // Preallocate buffer for SearchWithAnchoring() + RAPIDJSON_ASSERT(stateSet_ == 0); + if (stateCount_ > 0) { + stateSet_ = static_cast(states_.GetAllocator().Malloc(GetStateSetSize())); + state0_.template Reserve(stateCount_); + state1_.template Reserve(stateCount_); + } + } + + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { + State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; + } + + void PushOperand(Stack& operandStack, unsigned codepoint) { + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); + } + + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { + if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; + } + + SizeType Append(SizeType l1, SizeType l2) { + SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; + } + + void Patch(SizeType l, SizeType s) { + for (SizeType next; l != kRegexInvalidState; l = next) { + next = GetState(l).out; + GetState(l).out = s; + } + } + + bool Eval(Stack& operandStack, Operator op) { + switch (op) { + case kConcatenation: + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); + return true; + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + default: + RAPIDJSON_ASSERT(op == kOneOrMore); + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + } + } + + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { + RAPIDJSON_ASSERT(n <= m); + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) { + if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else { + Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? + } + return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) { + CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; + } + + static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } + + void CloneTopOperand(Stack& operandStack) { + const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) { + if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; + } + + template + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') { + if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; + } + + template + bool ParseRange(DecodedStream& ds, SizeType* range) { + bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) { + if (isBegin) { + isBegin = false; + if (codepoint == '^') { + negate = true; + continue; + } + } + + switch (codepoint) { + case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) { // Add trailing '-' + SizeType r = NewRange('-'); + RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; + } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; + + case '\\': + if (ds.Peek() == 'b') { + ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) { + case 1: + if (codepoint == '-') { + step++; + break; + } + // fall through to step 0 for other characters + + case 0: + { + SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; + } + } + } + return false; + } + + SizeType NewRange(unsigned codepoint) { + Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; + } + + template + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + unsigned codepoint; + switch (codepoint = ds.Take()) { + case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } + } + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const { + RAPIDJSON_ASSERT(IsValid()); + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == kAnyCharacterClass || + (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) const { + RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) { + stateSet_[index >> 5] |= (1 << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = GetRange(rangeIndex); + if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + Stack states_; + Stack ranges_; + SizeType root_; + SizeType stateCount_; + SizeType rangeCount_; + + static const unsigned kInfinityQuantifier = ~0u; + + // For SearchWithAnchoring() + uint32_t* stateSet_; // allocated by states_.GetAllocator() + mutable Stack state0_; + mutable Stack state1_; + bool anchorBegin_; + bool anchorEnd_; +}; + +typedef GenericRegex > Regex; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/stack.h b/ext/librethinkdbxx/src/rapidjson/internal/stack.h new file mode 100644 index 00000000..022c9aab --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/stack.h @@ -0,0 +1,230 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_STACK_H_ +#define RAPIDJSON_INTERNAL_STACK_H_ + +#include "../allocators.h" +#include "swap.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// Stack + +//! A type-unsafe stack for storing different types of data. +/*! \tparam Allocator Allocator for allocating stack memory. +*/ +template +class Stack { +public: + // Optimization note: Do not allocate memory for stack_ in constructor. + // Do it lazily when first Push() -> Expand() -> Resize(). + Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack(Stack&& rhs) + : allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(rhs.stack_), + stackTop_(rhs.stackTop_), + stackEnd_(rhs.stackEnd_), + initialCapacity_(rhs.initialCapacity_) + { + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } +#endif + + ~Stack() { + Destroy(); + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Stack& operator=(Stack&& rhs) { + if (&rhs != this) + { + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = rhs.stack_; + stackTop_ = rhs.stackTop_; + stackEnd_ = rhs.stackEnd_; + initialCapacity_ = rhs.initialCapacity_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } + return *this; + } +#endif + + void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(stack_, rhs.stack_); + internal::Swap(stackTop_, rhs.stackTop_); + internal::Swap(stackEnd_, rhs.stackEnd_); + internal::Swap(initialCapacity_, rhs.initialCapacity_); + } + + void Clear() { stackTop_ = stack_; } + + void ShrinkToFit() { + if (Empty()) { + // If the stack is empty, completely deallocate the memory. + Allocator::Free(stack_); + stack_ = 0; + stackTop_ = 0; + stackEnd_ = 0; + } + else + Resize(GetSize()); + } + + // Optimization note: try to minimize the size of this function for force inline. + // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. + template + RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { + // Expand the stack if needed + if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) + Expand(count); + } + + template + RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { + Reserve(count); + return PushUnsafe(count); + } + + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { + RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; + return ret; + } + + template + T* Pop(size_t count) { + RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); + stackTop_ -= count * sizeof(T); + return reinterpret_cast(stackTop_); + } + + template + T* Top() { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + const T* Top() const { + RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* End() { return reinterpret_cast(stackTop_); } + + template + const T* End() const { return reinterpret_cast(stackTop_); } + + template + T* Bottom() { return reinterpret_cast(stack_); } + + template + const T* Bottom() const { return reinterpret_cast(stack_); } + + bool HasAllocator() const { + return allocator_ != 0; + } + + Allocator& GetAllocator() { + RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + bool Empty() const { return stackTop_ == stack_; } + size_t GetSize() const { return static_cast(stackTop_ - stack_); } + size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } + +private: + template + void Expand(size_t count) { + // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. + size_t newCapacity; + if (stack_ == 0) { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + newCapacity = initialCapacity_; + } else { + newCapacity = GetCapacity(); + newCapacity += (newCapacity + 1) / 2; + } + size_t newSize = GetSize() + sizeof(T) * count; + if (newCapacity < newSize) + newCapacity = newSize; + + Resize(newCapacity); + } + + void Resize(size_t newCapacity) { + const size_t size = GetSize(); // Backup the current size + stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); + stackTop_ = stack_ + size; + stackEnd_ = stack_ + newCapacity; + } + + void Destroy() { + Allocator::Free(stack_); + RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack + } + + // Prohibit copy constructor & assignment operator. + Stack(const Stack&); + Stack& operator=(const Stack&); + + Allocator* allocator_; + Allocator* ownAllocator_; + char *stack_; + char *stackTop_; + char *stackEnd_; + size_t initialCapacity_; +}; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_STACK_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h b/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h new file mode 100644 index 00000000..2edfae52 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/strfunc.h @@ -0,0 +1,55 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ +#define RAPIDJSON_INTERNAL_STRFUNC_H_ + +#include "../stream.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Custom strlen() which works on different character types. +/*! \tparam Ch Character type (e.g. char, wchar_t, short) + \param s Null-terminated input string. + \return Number of characters in the string. + \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. +*/ +template +inline SizeType StrLen(const Ch* s) { + const Ch* p = s; + while (*p) ++p; + return SizeType(p - s); +} + +//! Returns number of code points in a encoded string. +template +bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { + GenericStringStream is(s); + const typename Encoding::Ch* end = s + length; + SizeType count = 0; + while (is.src_ < end) { + unsigned codepoint; + if (!Encoding::Decode(is, &codepoint)) + return false; + count++; + } + *outCount = count; + return true; +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_INTERNAL_STRFUNC_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/strtod.h b/ext/librethinkdbxx/src/rapidjson/internal/strtod.h new file mode 100644 index 00000000..289c413b --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/strtod.h @@ -0,0 +1,269 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_STRTOD_ +#define RAPIDJSON_STRTOD_ + +#include "ieee754.h" +#include "biginteger.h" +#include "diyfp.h" +#include "pow10.h" + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline double FastPath(double significand, int exp) { + if (exp < -308) + return 0.0; + else if (exp >= 0) + return significand * internal::Pow10(exp); + else + return significand / internal::Pow10(-exp); +} + +inline double StrtodNormalPrecision(double d, int p) { + if (p < -308) { + // Prevent expSum < -308, making Pow10(p) = 0 + d = FastPath(d, -308); + d = FastPath(d, p + 308); + } + else + d = FastPath(d, p); + return d; +} + +template +inline T Min3(T a, T b, T c) { + T m = a; + if (m > b) m = b; + if (m > c) m = c; + return m; +} + +inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { + const Double db(b); + const uint64_t bInt = db.IntegerSignificand(); + const int bExp = db.IntegerExponent(); + const int hExp = bExp - 1; + + int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; + + // Adjust for decimal exponent + if (dExp >= 0) { + dS_Exp2 += dExp; + dS_Exp5 += dExp; + } + else { + bS_Exp2 -= dExp; + bS_Exp5 -= dExp; + hS_Exp2 -= dExp; + hS_Exp5 -= dExp; + } + + // Adjust for binary exponent + if (bExp >= 0) + bS_Exp2 += bExp; + else { + dS_Exp2 -= bExp; + hS_Exp2 -= bExp; + } + + // Adjust for half ulp exponent + if (hExp >= 0) + hS_Exp2 += hExp; + else { + dS_Exp2 -= hExp; + bS_Exp2 -= hExp; + } + + // Remove common power of two factor from all three scaled values + int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); + dS_Exp2 -= common_Exp2; + bS_Exp2 -= common_Exp2; + hS_Exp2 -= common_Exp2; + + BigInteger dS = d; + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); + + BigInteger bS(bInt); + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); + + BigInteger hS(1); + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); + + BigInteger delta(0); + dS.Difference(bS, &delta); + + return delta.Compare(hS); +} + +inline bool StrtodFast(double d, int p, double* result) { + // Use fast path for string-to-double conversion if possible + // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + if (p > 22 && p < 22 + 16) { + // Fast Path Cases In Disguise + d *= internal::Pow10(p - 22); + p = 22; + } + + if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 + *result = FastPath(d, p); + return true; + } + else + return false; +} + +// Compute an approximation and see if it is within 1/2 ULP +inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { + uint64_t significand = 0; + size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 + for (; i < length; i++) { + if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) + break; + significand = significand * 10u + static_cast(decimals[i] - '0'); + } + + if (i < length && decimals[i] >= '5') // Rounding + significand++; + + size_t remaining = length - i; + const unsigned kUlpShift = 3; + const unsigned kUlp = 1 << kUlpShift; + int64_t error = (remaining == 0) ? 0 : kUlp / 2; + + DiyFp v(significand, 0); + v = v.Normalize(); + error <<= -v.e; + + const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; + + int actualExp; + DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); + if (actualExp != dExp) { + static const DiyFp kPow10[] = { + DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 + DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 + DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 + DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 + DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 + DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 + DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 + }; + int adjustment = dExp - actualExp - 1; + RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); + v = v * kPow10[adjustment]; + if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit + error += kUlp / 2; + } + + v = v * cachedPower; + + error += kUlp + (error == 0 ? 0 : 1); + + const int oldExp = v.e; + v = v.Normalize(); + error <<= oldExp - v.e; + + const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + unsigned precisionSize = 64 - effectiveSignificandSize; + if (precisionSize + kUlpShift >= 64) { + unsigned scaleExp = (precisionSize + kUlpShift) - 63; + v.f >>= scaleExp; + v.e += scaleExp; + error = (error >> scaleExp) + 1 + static_cast(kUlp); + precisionSize -= scaleExp; + } + + DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); + const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; + const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; + if (precisionBits >= halfWay + static_cast(error)) { + rounded.f++; + if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) + rounded.f >>= 1; + rounded.e++; + } + } + + *result = rounded.ToDouble(); + + return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); +} + +inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { + const BigInteger dInt(decimals, length); + const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; + Double a(approx); + int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); + if (cmp < 0) + return a.Value(); // within half ULP + else if (cmp == 0) { + // Round towards even + if (a.Significand() & 1) + return a.NextPositiveDouble(); + else + return a.Value(); + } + else // adjustment + return a.NextPositiveDouble(); +} + +inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { + RAPIDJSON_ASSERT(d >= 0.0); + RAPIDJSON_ASSERT(length >= 1); + + double result; + if (StrtodFast(d, p, &result)) + return result; + + // Trim leading zeros + while (*decimals == '0' && length > 1) { + length--; + decimals++; + decimalPosition--; + } + + // Trim trailing zeros + while (decimals[length - 1] == '0' && length > 1) { + length--; + decimalPosition--; + exp++; + } + + // Trim right-most digits + const int kMaxDecimalDigit = 780; + if (static_cast(length) > kMaxDecimalDigit) { + int delta = (static_cast(length) - kMaxDecimalDigit); + exp += delta; + decimalPosition -= static_cast(delta); + length = kMaxDecimalDigit; + } + + // If too small, underflow to zero + if (int(length) + exp < -324) + return 0.0; + + if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) + return result; + + // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison + return StrtodBigInteger(result, decimals, length, decimalPosition, exp); +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_STRTOD_ diff --git a/ext/librethinkdbxx/src/rapidjson/internal/swap.h b/ext/librethinkdbxx/src/rapidjson/internal/swap.h new file mode 100644 index 00000000..666e49f9 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/internal/swap.h @@ -0,0 +1,46 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_SWAP_H_ +#define RAPIDJSON_INTERNAL_SWAP_H_ + +#include "../rapidjson.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +//! Custom swap() to avoid dependency on C++ header +/*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. + \note This has the same semantics as std::swap(). +*/ +template +inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { + T tmp = a; + a = b; + b = tmp; +} + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_SWAP_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h b/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h new file mode 100644 index 00000000..f5fe2897 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/istreamwrapper.h @@ -0,0 +1,115 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_ISTREAMWRAPPER_H_ +#define RAPIDJSON_ISTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::istringstream + - \c std::stringstream + - \c std::wistringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wifstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_istream. +*/ + +template +class BasicIStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} + + Ch Peek() const { + typename StreamType::int_type c = stream_.peek(); + return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : '\0'; + } + + Ch Take() { + typename StreamType::int_type c = stream_.get(); + if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { + count_++; + return static_cast(c); + } + else + return '\0'; + } + + // tellg() may return -1 when failed. So we count by ourself. + size_t Tell() const { return count_; } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. + int i; + bool hasError = false; + for (i = 0; i < 4; ++i) { + typename StreamType::int_type c = stream_.get(); + if (c == StreamType::traits_type::eof()) { + hasError = true; + stream_.clear(); + break; + } + peekBuffer_[i] = static_cast(c); + } + for (--i; i >= 0; --i) + stream_.putback(peekBuffer_[i]); + return !hasError ? peekBuffer_ : 0; + } + +private: + BasicIStreamWrapper(const BasicIStreamWrapper&); + BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); + + StreamType& stream_; + size_t count_; //!< Number of characters read. Note: + mutable Ch peekBuffer_[4]; +}; + +typedef BasicIStreamWrapper IStreamWrapper; +typedef BasicIStreamWrapper WIStreamWrapper; + +#if defined(__clang__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_ISTREAMWRAPPER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/memorybuffer.h b/ext/librethinkdbxx/src/rapidjson/memorybuffer.h new file mode 100644 index 00000000..39bee1de --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/memorybuffer.h @@ -0,0 +1,70 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_MEMORYBUFFER_H_ +#define RAPIDJSON_MEMORYBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output byte stream. +/*! + This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. + + It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. + + Differences between MemoryBuffer and StringBuffer: + 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. + 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. + + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +struct GenericMemoryBuffer { + typedef char Ch; // byte + + GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { stack_.ShrinkToFit(); } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetBuffer() const { + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; +}; + +typedef GenericMemoryBuffer<> MemoryBuffer; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { + std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); +} + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/memorystream.h b/ext/librethinkdbxx/src/rapidjson/memorystream.h new file mode 100644 index 00000000..1d71d8a4 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/memorystream.h @@ -0,0 +1,71 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_MEMORYSTREAM_H_ +#define RAPIDJSON_MEMORYSTREAM_H_ + +#include "stream.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory input byte stream. +/*! + This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. + + It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. + + Differences between MemoryStream and StringStream: + 1. StringStream has encoding but MemoryStream is a byte stream. + 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. + 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). + \note implements Stream concept +*/ +struct MemoryStream { + typedef char Ch; // byte + + MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} + + Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } + Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } + size_t Tell() const { return static_cast(src_ - begin_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return Tell() + 4 <= size_ ? src_ : 0; + } + + const Ch* src_; //!< Current read position. + const Ch* begin_; //!< Original head of the string. + const Ch* end_; //!< End of stream. + size_t size_; //!< Size of the stream. +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h b/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h new file mode 100644 index 00000000..18111286 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/msinttypes/inttypes.h @@ -0,0 +1,316 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include "stdint.h" + +// miloyip: VC supports inttypes.h since VC2013 +#if _MSC_VER >= 1800 +#include +#else + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "I32d" +#define PRIi32 "I32i" +#define PRIdLEAST32 "I32d" +#define PRIiLEAST32 "I32i" +#define PRIdFAST32 "I32d" +#define PRIiFAST32 "I32i" + +#define PRId64 "I64d" +#define PRIi64 "I64i" +#define PRIdLEAST64 "I64d" +#define PRIiLEAST64 "I64i" +#define PRIdFAST64 "I64d" +#define PRIiFAST64 "I64i" + +#define PRIdMAX "I64d" +#define PRIiMAX "I64i" + +#define PRIdPTR "Id" +#define PRIiPTR "Ii" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "I32o" +#define PRIu32 "I32u" +#define PRIx32 "I32x" +#define PRIX32 "I32X" +#define PRIoLEAST32 "I32o" +#define PRIuLEAST32 "I32u" +#define PRIxLEAST32 "I32x" +#define PRIXLEAST32 "I32X" +#define PRIoFAST32 "I32o" +#define PRIuFAST32 "I32u" +#define PRIxFAST32 "I32x" +#define PRIXFAST32 "I32X" + +#define PRIo64 "I64o" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRIX64 "I64X" +#define PRIoLEAST64 "I64o" +#define PRIuLEAST64 "I64u" +#define PRIxLEAST64 "I64x" +#define PRIXLEAST64 "I64X" +#define PRIoFAST64 "I64o" +#define PRIuFAST64 "I64u" +#define PRIxFAST64 "I64x" +#define PRIXFAST64 "I64X" + +#define PRIoMAX "I64o" +#define PRIuMAX "I64u" +#define PRIxMAX "I64x" +#define PRIXMAX "I64X" + +#define PRIoPTR "Io" +#define PRIuPTR "Iu" +#define PRIxPTR "Ix" +#define PRIXPTR "IX" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + +#endif // _MSC_VER >= 1800 + +#endif // _MSC_INTTYPES_H_ ] diff --git a/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h b/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h new file mode 100644 index 00000000..3d4477b9 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/msinttypes/stdint.h @@ -0,0 +1,300 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. +#if _MSC_VER >= 1600 // [ +#include + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +#undef INT8_C +#undef INT16_C +#undef INT32_C +#undef INT64_C +#undef UINT8_C +#undef UINT16_C +#undef UINT32_C +#undef UINT64_C + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#else // ] _MSC_VER >= 1700 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we have to wrap include with 'extern "C++" {}' +// or compiler would give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#if defined(__cplusplus) && !defined(_M_ARM) +extern "C" { +#endif +# include +#if defined(__cplusplus) && !defined(_M_ARM) +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff --git a/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h b/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h new file mode 100644 index 00000000..6f4667c0 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/ostreamwrapper.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ +#define RAPIDJSON_OSTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::ostringstream + - \c std::stringstream + - \c std::wpstringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wofstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_ostream. +*/ + +template +class BasicOStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + + void Put(Ch c) { + stream_.put(c); + } + + void Flush() { + stream_.flush(); + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + + StreamType& stream_; +}; + +typedef BasicOStreamWrapper OStreamWrapper; +typedef BasicOStreamWrapper WOStreamWrapper; + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/pointer.h b/ext/librethinkdbxx/src/rapidjson/pointer.h new file mode 100644 index 00000000..0206ac1c --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/pointer.h @@ -0,0 +1,1358 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_POINTER_H_ +#define RAPIDJSON_POINTER_H_ + +#include "document.h" +#include "internal/itoa.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token + +//! Error code of parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode +*/ +enum PointerParseErrorCode { + kPointerParseErrorNone = 0, //!< The parse is successful + + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericPointer + +//! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. +/*! + This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" + (https://tools.ietf.org/html/rfc6901). + + A JSON pointer is for identifying a specific value in a JSON document + (GenericDocument). It can simplify coding of DOM tree manipulation, because it + can access multiple-level depth of DOM tree with single API call. + + After it parses a string representation (e.g. "/foo/0" or URI fragment + representation (e.g. "#/foo/0") into its internal representation (tokens), + it can be used to resolve a specific value in multiple documents, or sub-tree + of documents. + + Contrary to GenericValue, Pointer can be copy constructed and copy assigned. + Apart from assignment, a Pointer cannot be modified after construction. + + Although Pointer is very convenient, please aware that constructing Pointer + involves parsing and dynamic memory allocation. A special constructor with user- + supplied tokens eliminates these. + + GenericPointer depends on GenericDocument and GenericValue. + + \tparam ValueType The value type of the DOM tree. E.g. GenericValue > + \tparam Allocator The allocator type for allocating memory for internal representation. + + \note GenericPointer uses same encoding of ValueType. + However, Allocator of GenericPointer is independent of Allocator of Value. +*/ +template +class GenericPointer { +public: + typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value + + //! A token is the basic units of internal representation. + /*! + A JSON pointer string representation "/foo/123" is parsed to two tokens: + "foo" and 123. 123 will be represented in both numeric form and string form. + They are resolved according to the actual value type (object or array). + + For token that are not numbers, or the numeric value is out of bound + (greater than limits of SizeType), they are only treated as string form + (i.e. the token's index will be equal to kPointerInvalidIndex). + + This struct is public so that user can create a Pointer without parsing and + allocation, using a special constructor. + */ + struct Token { + const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. + SizeType length; //!< Length of the name. + SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. + }; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor. + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A null-terminated, string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + */ + explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, internal::StrLen(source)); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source.c_str(), source.size()); + } +#endif + + //! Constructor that parses a string or URI fragment representation, with length of the source string. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param length Length of source. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Slightly faster than the overload without length. + */ + GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, length); + } + + //! Constructor with user-supplied tokens. + /*! + This constructor let user supplies const array of tokens. + This prevents the parsing process and eliminates allocation. + This is preferred for memory constrained environments. + + \param tokens An constant array of tokens representing the JSON pointer. + \param tokenCount Number of tokens. + + \b Example + \code + #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } + #define INDEX(i) { #i, sizeof(#i) - 1, i } + + static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; + static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + // Equivalent to static const Pointer p("/foo/123"); + + #undef NAME + #undef INDEX + \endcode + */ + GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Destructor. + ~GenericPointer() { + if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. + Allocator::Free(tokens_); + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Assignment operator. + GenericPointer& operator=(const GenericPointer& rhs) { + if (this != &rhs) { + // Do not delete ownAllcator + if (nameBuffer_) + Allocator::Free(tokens_); + + tokenCount_ = rhs.tokenCount_; + parseErrorOffset_ = rhs.parseErrorOffset_; + parseErrorCode_ = rhs.parseErrorCode_; + + if (rhs.nameBuffer_) + CopyFromRaw(rhs); // Normally parsed tokens. + else { + tokens_ = rhs.tokens_; // User supplied const tokens. + nameBuffer_ = 0; + } + } + return *this; + } + + //@} + + //!@name Append token + //@{ + + //! Append a token and return a new Pointer + /*! + \param token Token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Token& token, Allocator* allocator = 0) const { + GenericPointer r; + r.allocator_ = allocator; + Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); + std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); + r.tokens_[tokenCount_].name = p; + r.tokens_[tokenCount_].length = token.length; + r.tokens_[tokenCount_].index = token.index; + return r; + } + + //! Append a name token with length, and return a new Pointer + /*! + \param name Name to be appended. + \param length Length of name. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { + Token token = { name, length, kPointerInvalidIndex }; + return Append(token, allocator); + } + + //! Append a name token without length, and return a new Pointer + /*! + \param name Name (const Ch*) to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) + Append(T* name, Allocator* allocator = 0) const { + return Append(name, StrLen(name), allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Append a name token, and return a new Pointer + /*! + \param name Name to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { + return Append(name.c_str(), static_cast(name.size()), allocator); + } +#endif + + //! Append a index token, and return a new Pointer + /*! + \param index Index to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(SizeType index, Allocator* allocator = 0) const { + char buffer[21]; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); + buffer[length] = '\0'; + + if (sizeof(Ch) == 1) { + Token token = { reinterpret_cast(buffer), length, index }; + return Append(token, allocator); + } + else { + Ch name[21]; + for (size_t i = 0; i <= length; i++) + name[i] = buffer[i]; + Token token = { name, length, index }; + return Append(token, allocator); + } + } + + //! Append a token by value, and return a new Pointer + /*! + \param token token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { + if (token.IsString()) + return Append(token.GetString(), token.GetStringLength(), allocator); + else { + RAPIDJSON_ASSERT(token.IsUint64()); + RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); + return Append(static_cast(token.GetUint64()), allocator); + } + } + + //!@name Handling Parse Error + //@{ + + //! Check whether this is a valid pointer. + bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } + + //! Get the parsing error offset in code unit. + size_t GetParseErrorOffset() const { return parseErrorOffset_; } + + //! Get the parsing error code. + PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } + + //@} + + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } + + //!@name Tokens + //@{ + + //! Get the token array (const version only). + const Token* GetTokens() const { return tokens_; } + + //! Get the number of tokens. + size_t GetTokenCount() const { return tokenCount_; } + + //@} + + //!@name Equality/inequality operators + //@{ + + //! Equality operator. + /*! + \note When any pointers are invalid, always returns false. + */ + bool operator==(const GenericPointer& rhs) const { + if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) + return false; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index || + tokens_[i].length != rhs.tokens_[i].length || + (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) + { + return false; + } + } + + return true; + } + + //! Inequality operator. + /*! + \note When any pointers are invalid, always returns true. + */ + bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } + + //@} + + //!@name Stringify + //@{ + + //! Stringify the pointer into string representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + return Stringify(os); + } + + //! Stringify the pointer into URI fragment representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool StringifyUriFragment(OutputStream& os) const { + return Stringify(os); + } + + //@} + + //!@name Create value + //@{ + + //! Create a value in a subtree. + /*! + If the value is not exist, it creates all parent values and a JSON Null value. + So it always succeed and return the newly created or existing value. + + Remind that it may change types of parents according to tokens, so it + potentially removes previously stored values. For example, if a document + was an array, and "/foo" is used to create a value, then the document + will be changed to an object, and all existing array elements are lost. + + \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created (a JSON Null value), or already exists value. + */ + ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { + RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + bool exist = true; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + if (v->IsArray() && t->name[0] == '-' && t->length == 1) { + v->PushBack(ValueType().Move(), allocator); + v = &((*v)[v->Size() - 1]); + exist = false; + } + else { + if (t->index == kPointerInvalidIndex) { // must be object name + if (!v->IsObject()) + v->SetObject(); // Change to Object + } + else { // object name or array index + if (!v->IsArray() && !v->IsObject()) + v->SetArray(); // Change to Array + } + + if (v->IsArray()) { + if (t->index >= v->Size()) { + v->Reserve(t->index + 1, allocator); + while (t->index >= v->Size()) + v->PushBack(ValueType().Move(), allocator); + exist = false; + } + v = &((*v)[t->index]); + } + else { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) { + v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); + v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end + exist = false; + } + else + v = &m->value; + } + } + } + + if (alreadyExist) + *alreadyExist = exist; + + return *v; + } + + //! Creates a value in a document. + /*! + \param document A document to be resolved. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created, or already exists value. + */ + template + ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { + return Create(document, document.GetAllocator(), alreadyExist); + } + + //@} + + //!@name Query value + //@{ + + //! Query a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. + \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. + */ + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { + RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + break; + v = &m->value; + } + continue; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + break; + v = &((*v)[t->index]); + continue; + default: + break; + } + + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; + } + return v; + } + + //! Query a const value in a const subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Pointer to the value if it can be resolved. Otherwise null. + */ + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { + return Get(const_cast(root), unresolvedTokenIndex); + } + + //@} + + //!@name Query a value with default + //@{ + + //! Query a value in a subtree with default value. + /*! + Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. + So that this function always succeed. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param defaultValue Default value to be cloned if the value was not exists. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); + } + + //! Query a value in a subtree with default null-terminated string. + ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Query a value in a subtree with default std::basic_string. + ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } +#endif + + //! Query a value in a subtree with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { + return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); + } + + //! Query a value in a document with default value. + template + ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //! Query a value in a document with default null-terminated string. + template + ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Query a value in a document with default std::basic_string. + template + ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } +#endif + + //! Query a value in a document with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(GenericDocument& document, T defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //@} + + //!@name Set a value + //@{ + + //! Set a value in a subtree, with move semantics. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be set. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = value; + } + + //! Set a value in a subtree, with copy semantics. + ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).CopyFrom(value, allocator); + } + + //! Set a null-terminated string in a subtree. + ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Set a std::basic_string in a subtree. + ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } +#endif + + //! Set a primitive value in a subtree. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value).Move(); + } + + //! Set a value in a document, with move semantics. + template + ValueType& Set(GenericDocument& document, ValueType& value) const { + return Create(document) = value; + } + + //! Set a value in a document, with copy semantics. + template + ValueType& Set(GenericDocument& document, const ValueType& value) const { + return Create(document).CopyFrom(value, document.GetAllocator()); + } + + //! Set a null-terminated string in a document. + template + ValueType& Set(GenericDocument& document, const Ch* value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Sets a std::basic_string in a document. + template + ValueType& Set(GenericDocument& document, const std::basic_string& value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } +#endif + + //! Set a primitive value in a document. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(GenericDocument& document, T value) const { + return Create(document) = value; + } + + //@} + + //!@name Swap a value + //@{ + + //! Swap a value with a value in a subtree. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be swapped. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).Swap(value); + } + + //! Swap a value with a value in a document. + template + ValueType& Swap(GenericDocument& document, ValueType& value) const { + return Create(document).Swap(value); + } + + //@} + + //! Erase a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Whether the resolved value is found and erased. + + \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. + */ + bool Erase(ValueType& root) const { + RAPIDJSON_ASSERT(IsValid()); + if (tokenCount_ == 0) // Cannot erase the root + return false; + + ValueType* v = &root; + const Token* last = tokens_ + (tokenCount_ - 1); + for (const Token *t = tokens_; t != last; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + return false; + v = &m->value; + } + break; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + return false; + v = &((*v)[t->index]); + break; + default: + return false; + } + } + + switch (v->GetType()) { + case kObjectType: + return v->EraseMember(GenericStringRef(last->name, last->length)); + case kArrayType: + if (last->index == kPointerInvalidIndex || last->index >= v->Size()) + return false; + v->Erase(v->Begin() + last->index); + return true; + default: + return false; + } + } + +private: + //! Clone the content from rhs to this. + /*! + \param rhs Source pointer. + \param extraToken Extra tokens to be allocated. + \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. + \return Start of non-occupied name buffer, for storing extra names. + */ + Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { + if (!allocator_) // allocator is independently owned. + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens + for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) + nameBufferSize += t->length; + + tokenCount_ = rhs.tokenCount_ + extraToken; + tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); + nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + if (rhs.tokenCount_ > 0) { + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + } + if (nameBufferSize > 0) { + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + } + + // Adjust pointers to name buffer + std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; + for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) + t->name += diff; + + return nameBuffer_ + nameBufferSize; + } + + //! Check whether a character should be percent-encoded. + /*! + According to RFC 3986 2.3 Unreserved Characters. + \param c The character (code unit) to be tested. + */ + bool NeedPercentEncode(Ch c) const { + return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); + } + + //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation + /*! + \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. + \param length Length of the source string. + \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. + */ +#endif + void Parse(const Ch* source, size_t length) { + RAPIDJSON_ASSERT(source != NULL); + RAPIDJSON_ASSERT(nameBuffer_ == 0); + RAPIDJSON_ASSERT(tokens_ == 0); + + // Create own allocator if user did not supply. + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + // Count number of '/' as tokenCount + tokenCount_ = 0; + for (const Ch* s = source; s != source + length; s++) + if (*s == '/') + tokenCount_++; + + Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); + Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + size_t i = 0; + + // Detect if it is a URI fragment + bool uriFragment = false; + if (source[i] == '#') { + uriFragment = true; + i++; + } + + if (i != length && source[i] != '/') { + parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; + goto error; + } + + while (i < length) { + RAPIDJSON_ASSERT(source[i] == '/'); + i++; // consumes '/' + + token->name = name; + bool isNumber = true; + + while (i < length && source[i] != '/') { + Ch c = source[i]; + if (uriFragment) { + // Decoding percent-encoding for URI fragment + if (c == '%') { + PercentDecodeStream is(&source[i], source + length); + GenericInsituStringStream os(name); + Ch* begin = os.PutBegin(); + if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { + parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; + goto error; + } + size_t len = os.PutEnd(begin); + i += is.Tell() - 1; + if (len == 1) + c = *name; + else { + name += len; + isNumber = false; + i++; + continue; + } + } + else if (NeedPercentEncode(c)) { + parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; + goto error; + } + } + + i++; + + // Escaping "~0" -> '~', "~1" -> '/' + if (c == '~') { + if (i < length) { + c = source[i]; + if (c == '0') c = '~'; + else if (c == '1') c = '/'; + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + i++; + } + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + } + + // First check for index: all of characters are digit + if (c < '0' || c > '9') + isNumber = false; + + *name++ = c; + } + token->length = static_cast(name - token->name); + if (token->length == 0) + isNumber = false; + *name++ = '\0'; // Null terminator + + // Second check for index: more than one digit cannot have leading zero + if (isNumber && token->length > 1 && token->name[0] == '0') + isNumber = false; + + // String to SizeType conversion + SizeType n = 0; + if (isNumber) { + for (size_t j = 0; j < token->length; j++) { + SizeType m = n * 10 + static_cast(token->name[j] - '0'); + if (m < n) { // overflow detection + isNumber = false; + break; + } + n = m; + } + } + + token->index = isNumber ? n : kPointerInvalidIndex; + token++; + } + + RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer + parseErrorCode_ = kPointerParseErrorNone; + return; + + error: + Allocator::Free(tokens_); + nameBuffer_ = 0; + tokens_ = 0; + tokenCount_ = 0; + parseErrorOffset_ = i; + return; + } + + //! Stringify to string or URI fragment representation. + /*! + \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. + \tparam OutputStream type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + RAPIDJSON_ASSERT(IsValid()); + + if (uriFragment) + os.Put('#'); + + for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + os.Put('/'); + for (size_t j = 0; j < t->length; j++) { + Ch c = t->name[j]; + if (c == '~') { + os.Put('~'); + os.Put('0'); + } + else if (c == '/') { + os.Put('~'); + os.Put('1'); + } + else if (uriFragment && NeedPercentEncode(c)) { + // Transcode to UTF8 sequence + GenericStringStream source(&t->name[j]); + PercentEncodeStream target(os); + if (!Transcoder >().Validate(source, target)) + return false; + j += source.Tell() - 1; + } + else + os.Put(c); + } + } + return true; + } + + //! A helper stream for decoding a percent-encoded sequence into code unit. + /*! + This stream decodes %XY triplet into code unit (0-255). + If it encounters invalid characters, it sets output code unit as 0 and + mark invalid, and to be checked by IsValid(). + */ + class PercentDecodeStream { + public: + typedef typename ValueType::Ch Ch; + + //! Constructor + /*! + \param source Start of the stream + \param end Past-the-end of the stream. + */ + PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} + + Ch Take() { + if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet + valid_ = false; + return 0; + } + src_++; + Ch c = 0; + for (int j = 0; j < 2; j++) { + c = static_cast(c << 4); + Ch h = *src_; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); + else { + valid_ = false; + return 0; + } + src_++; + } + return c; + } + + size_t Tell() const { return static_cast(src_ - head_); } + bool IsValid() const { return valid_; } + + private: + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. + const Ch* end_; //!< Past-the-end position. + bool valid_; //!< Whether the parsing is valid. + }; + + //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. + template + class PercentEncodeStream { + public: + PercentEncodeStream(OutputStream& os) : os_(os) {} + void Put(char c) { // UTF-8 must be byte + unsigned char u = static_cast(c); + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + os_.Put('%'); + os_.Put(hexDigits[u >> 4]); + os_.Put(hexDigits[u & 15]); + } + private: + OutputStream& os_; + }; + + Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. + Allocator* ownAllocator_; //!< Allocator owned by this Pointer. + Ch* nameBuffer_; //!< A buffer containing all names in tokens. + Token* tokens_; //!< A list of tokens. + size_t tokenCount_; //!< Number of tokens in tokens_. + size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. + PointerParseErrorCode parseErrorCode_; //!< Parsing error code. +}; + +//! GenericPointer for Value (UTF-8, default allocator). +typedef GenericPointer Pointer; + +//!@name Helper functions for GenericPointer +//@{ + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { + return pointer.Create(root, a); +} + +template +typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Create(root, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { + return pointer.Create(document); +} + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Create(document); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { + return pointer.Set(document, value); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { + return pointer.Set(document, value); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { + return GenericPointer(source, N - 1).Set(document, value); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Swap(root, value, a); +} + +template +typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Swap(root, value, a); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Swap(document, value); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Swap(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +bool EraseValueByPointer(T& root, const GenericPointer& pointer) { + return pointer.Erase(root); +} + +template +bool EraseValueByPointer(T& root, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Erase(root); +} + +//@} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_POINTER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/prettywriter.h b/ext/librethinkdbxx/src/rapidjson/prettywriter.h new file mode 100644 index 00000000..75dc474f --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/prettywriter.h @@ -0,0 +1,249 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_PRETTYWRITER_H_ +#define RAPIDJSON_PRETTYWRITER_H_ + +#include "writer.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Combination of PrettyWriter format flags. +/*! \see PrettyWriter::SetFormatOptions + */ +enum PrettyFormatOptions { + kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. +}; + +//! Writer with indentation and spacing. +/*! + \tparam OutputStream Type of ouptut os. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class PrettyWriter : public Writer { +public: + typedef Writer Base; + typedef typename Base::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + + //! Set custom indentation. + /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). + \param indentCharCount Number of indent characters for each indentation level. + \note The default indentation is 4 spaces. + */ + PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { + RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); + indentChar_ = indentChar; + indentCharCount_ = indentCharCount; + return *this; + } + + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { + formatOptions_ = options; + return *this; + } + + /*! @name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + PrettyPrefix(kNumberType); + return Base::WriteString(str, length); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + PrettyPrefix(kStringType); + return Base::WriteString(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + PrettyPrefix(kObjectType); + new (Base::level_stack_.template Push()) typename Base::Level(false); + return Base::WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::WriteEndObject(); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + bool StartArray() { + PrettyPrefix(kArrayType); + new (Base::level_stack_.template Push()) typename Base::Level(true); + return Base::WriteStartArray(); + } + + bool EndArray(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::WriteEndArray(); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } + +protected: + void PrettyPrefix(Type type) { + (void)type; + if (Base::level_stack_.GetSize() != 0) { // this value is not at root + typename Base::Level* level = Base::level_stack_.template Top(); + + if (level->inArray) { + if (level->valueCount > 0) { + Base::os_->Put(','); // add comma if it is not the first element in array + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); + } + + if (!(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + } + else { // in object + if (level->valueCount > 0) { + if (level->valueCount % 2 == 0) { + Base::os_->Put(','); + Base::os_->Put('\n'); + } + else { + Base::os_->Put(':'); + Base::os_->Put(' '); + } + } + else + Base::os_->Put('\n'); + + if (level->valueCount % 2 == 0) + WriteIndent(); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. + Base::hasRoot_ = true; + } + } + + void WriteIndent() { + size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; + PutN(*Base::os_, static_cast(indentChar_), count); + } + + Ch indentChar_; + unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; + +private: + // Prohibit copy constructor & assignment operator. + PrettyWriter(const PrettyWriter&); + PrettyWriter& operator=(const PrettyWriter&); +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/rapidjson.h b/ext/librethinkdbxx/src/rapidjson/rapidjson.h new file mode 100644 index 00000000..d666f202 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/rapidjson.h @@ -0,0 +1,615 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_RAPIDJSON_H_ +#define RAPIDJSON_RAPIDJSON_H_ + +/*!\file rapidjson.h + \brief common definitions and configuration + + \see RAPIDJSON_CONFIG + */ + +/*! \defgroup RAPIDJSON_CONFIG RapidJSON configuration + \brief Configuration macros for library features + + Some RapidJSON features are configurable to adapt the library to a wide + variety of platforms, environments and usage scenarios. Most of the + features can be configured in terms of overriden or predefined + preprocessor macros at compile-time. + + Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. + + \note These macros should be given on the compiler command-line + (where applicable) to avoid inconsistent values when compiling + different translation units of a single application. + */ + +#include // malloc(), realloc(), free(), size_t +#include // memset(), memcpy(), memmove(), memcmp() + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_VERSION_STRING +// +// ALWAYS synchronize the following 3 macros with corresponding variables in /CMakeLists.txt. +// + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +// token stringification +#define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) +#define RAPIDJSON_DO_STRINGIFY(x) #x +//!@endcond + +/*! \def RAPIDJSON_MAJOR_VERSION + \ingroup RAPIDJSON_CONFIG + \brief Major version of RapidJSON in integer. +*/ +/*! \def RAPIDJSON_MINOR_VERSION + \ingroup RAPIDJSON_CONFIG + \brief Minor version of RapidJSON in integer. +*/ +/*! \def RAPIDJSON_PATCH_VERSION + \ingroup RAPIDJSON_CONFIG + \brief Patch version of RapidJSON in integer. +*/ +/*! \def RAPIDJSON_VERSION_STRING + \ingroup RAPIDJSON_CONFIG + \brief Version of RapidJSON in ".." string format. +*/ +#define RAPIDJSON_MAJOR_VERSION 1 +#define RAPIDJSON_MINOR_VERSION 0 +#define RAPIDJSON_PATCH_VERSION 2 +#define RAPIDJSON_VERSION_STRING \ + RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NAMESPACE_(BEGIN|END) +/*! \def RAPIDJSON_NAMESPACE + \ingroup RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace + + In order to avoid symbol clashes and/or "One Definition Rule" errors + between multiple inclusions of (different versions of) RapidJSON in + a single binary, users can customize the name of the main RapidJSON + namespace. + + In case of a single nesting level, defining \c RAPIDJSON_NAMESPACE + to a custom name (e.g. \c MyRapidJSON) is sufficient. If multiple + levels are needed, both \ref RAPIDJSON_NAMESPACE_BEGIN and \ref + RAPIDJSON_NAMESPACE_END need to be defined as well: + + \code + // in some .cpp file + #define RAPIDJSON_NAMESPACE my::rapidjson + #define RAPIDJSON_NAMESPACE_BEGIN namespace my { namespace rapidjson { + #define RAPIDJSON_NAMESPACE_END } } + #include "rapidjson/..." + \endcode + + \see rapidjson + */ +/*! \def RAPIDJSON_NAMESPACE_BEGIN + \ingroup RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace (opening expression) + \see RAPIDJSON_NAMESPACE +*/ +/*! \def RAPIDJSON_NAMESPACE_END + \ingroup RAPIDJSON_CONFIG + \brief provide custom rapidjson namespace (closing expression) + \see RAPIDJSON_NAMESPACE +*/ +#ifndef RAPIDJSON_NAMESPACE +#define RAPIDJSON_NAMESPACE rapidjson +#endif +#ifndef RAPIDJSON_NAMESPACE_BEGIN +#define RAPIDJSON_NAMESPACE_BEGIN namespace RAPIDJSON_NAMESPACE { +#endif +#ifndef RAPIDJSON_NAMESPACE_END +#define RAPIDJSON_NAMESPACE_END } +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_HAS_STDSTRING + +#ifndef RAPIDJSON_HAS_STDSTRING +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation +#else +#define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default +#endif +/*! \def RAPIDJSON_HAS_STDSTRING + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for \c std::string + + By defining this preprocessor symbol to \c 1, several convenience functions for using + \ref rapidjson::GenericValue with \c std::string are enabled, especially + for construction and comparison. + + \hideinitializer +*/ +#endif // !defined(RAPIDJSON_HAS_STDSTRING) + +#if RAPIDJSON_HAS_STDSTRING +#include +#endif // RAPIDJSON_HAS_STDSTRING + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NO_INT64DEFINE + +/*! \def RAPIDJSON_NO_INT64DEFINE + \ingroup RAPIDJSON_CONFIG + \brief Use external 64-bit integer types. + + RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types + to be available at global scope. + + If users have their own definition, define RAPIDJSON_NO_INT64DEFINE to + prevent RapidJSON from defining its own types. +*/ +#ifndef RAPIDJSON_NO_INT64DEFINE +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#include "msinttypes/stdint.h" +#include "msinttypes/inttypes.h" +#else +// Other compilers should have this. +#include +#include +#endif +//!@endcond +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_NO_INT64DEFINE +#endif +#endif // RAPIDJSON_NO_INT64TYPEDEF + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_FORCEINLINE + +#ifndef RAPIDJSON_FORCEINLINE +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#if defined(_MSC_VER) && defined(NDEBUG) +#define RAPIDJSON_FORCEINLINE __forceinline +#elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) +#define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) +#else +#define RAPIDJSON_FORCEINLINE +#endif +//!@endcond +#endif // RAPIDJSON_FORCEINLINE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ENDIAN +#define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine +#define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine + +//! Endianness of the machine. +/*! + \def RAPIDJSON_ENDIAN + \ingroup RAPIDJSON_CONFIG + + GCC 4.6 provided macro for detecting endianness of the target machine. But other + compilers may not have this. User can define RAPIDJSON_ENDIAN to either + \ref RAPIDJSON_LITTLEENDIAN or \ref RAPIDJSON_BIGENDIAN. + + Default detection implemented with reference to + \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html + \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp +*/ +#ifndef RAPIDJSON_ENDIAN +// Detect with GCC 4.6's macro +# ifdef __BYTE_ORDER__ +# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif // __BYTE_ORDER__ +// Detect with GLIBC's endian.h +# elif defined(__GLIBC__) +# include +# if (__BYTE_ORDER == __LITTLE_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif (__BYTE_ORDER == __BIG_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif // __GLIBC__ +// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro +# elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +// Detect with architecture macros +# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) +# define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN +# elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(_MSC_VER) && defined(_M_ARM) +# define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN +# elif defined(RAPIDJSON_DOXYGEN_RUNNING) +# define RAPIDJSON_ENDIAN +# else +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# endif +#endif // RAPIDJSON_ENDIAN + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_64BIT + +//! Whether using 64-bit architecture +#ifndef RAPIDJSON_64BIT +#if defined(__LP64__) || defined(_WIN64) || defined(__EMSCRIPTEN__) +#define RAPIDJSON_64BIT 1 +#else +#define RAPIDJSON_64BIT 0 +#endif +#endif // RAPIDJSON_64BIT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ALIGN + +//! Data alignment of the machine. +/*! \ingroup RAPIDJSON_CONFIG + \param x pointer to align + + Some machines require strict data alignment. Currently the default uses 4 bytes + alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. + User can customize by defining the RAPIDJSON_ALIGN function macro. +*/ +#ifndef RAPIDJSON_ALIGN +#if RAPIDJSON_64BIT == 1 +#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) +#else +#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) +#endif +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_UINT64_C2 + +//! Construct a 64-bit literal by a pair of 32-bit integer. +/*! + 64-bit literal with or without ULL suffix is prone to compiler warnings. + UINT64_C() is C macro which cause compilation problems. + Use this macro to define 64-bit constants by a pair of 32-bit integer. +*/ +#ifndef RAPIDJSON_UINT64_C2 +#define RAPIDJSON_UINT64_C2(high32, low32) ((static_cast(high32) << 32) | static_cast(low32)) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_48BITPOINTER_OPTIMIZATION + +//! Use only lower 48-bit address for some pointers. +/*! + \ingroup RAPIDJSON_CONFIG + + This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. + The higher 16-bit can be used for storing other data. + \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. +*/ +#ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 +#else +#define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 +#endif +#endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION + +#if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 +#if RAPIDJSON_64BIT != 1 +#error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 +#endif +#define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) +#define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) +#else +#define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) +#define RAPIDJSON_GETPOINTER(type, p) (p) +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD + +/*! \def RAPIDJSON_SIMD + \ingroup RAPIDJSON_CONFIG + \brief Enable SSE2/SSE4.2 optimization. + + RapidJSON supports optimized implementations for some parsing operations + based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible + processors. + + To enable these optimizations, two different symbols can be defined; + \code + // Enable SSE2 optimization. + #define RAPIDJSON_SSE2 + + // Enable SSE4.2 optimization. + #define RAPIDJSON_SSE42 + \endcode + + \c RAPIDJSON_SSE42 takes precedence, if both are defined. + + If any of these symbols is defined, RapidJSON defines the macro + \c RAPIDJSON_SIMD to indicate the availability of the optimized code. +*/ +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ + || defined(RAPIDJSON_DOXYGEN_RUNNING) +#define RAPIDJSON_SIMD +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NO_SIZETYPEDEFINE + +#ifndef RAPIDJSON_NO_SIZETYPEDEFINE +/*! \def RAPIDJSON_NO_SIZETYPEDEFINE + \ingroup RAPIDJSON_CONFIG + \brief User-provided \c SizeType definition. + + In order to avoid using 32-bit size types for indexing strings and arrays, + define this preprocessor symbol and provide the type rapidjson::SizeType + before including RapidJSON: + \code + #define RAPIDJSON_NO_SIZETYPEDEFINE + namespace rapidjson { typedef ::std::size_t SizeType; } + #include "rapidjson/..." + \endcode + + \see rapidjson::SizeType +*/ +#ifdef RAPIDJSON_DOXYGEN_RUNNING +#define RAPIDJSON_NO_SIZETYPEDEFINE +#endif +RAPIDJSON_NAMESPACE_BEGIN +//! Size type (for string lengths, array sizes, etc.) +/*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, + instead of using \c size_t. Users may override the SizeType by defining + \ref RAPIDJSON_NO_SIZETYPEDEFINE. +*/ +typedef unsigned SizeType; +RAPIDJSON_NAMESPACE_END +#endif + +// always import std::size_t to rapidjson namespace +RAPIDJSON_NAMESPACE_BEGIN +using std::size_t; +RAPIDJSON_NAMESPACE_END + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_ASSERT + +//! Assertion. +/*! \ingroup RAPIDJSON_CONFIG + By default, rapidjson uses C \c assert() for internal assertions. + User can override it by defining RAPIDJSON_ASSERT(x) macro. + + \note Parsing errors are handled and can be customized by the + \ref RAPIDJSON_ERRORS APIs. +*/ +#ifndef RAPIDJSON_ASSERT +#include +#define RAPIDJSON_ASSERT(x) assert(x) +#endif // RAPIDJSON_ASSERT + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_STATIC_ASSERT + +// Adopt from boost +#ifndef RAPIDJSON_STATIC_ASSERT +#ifndef __clang__ +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#endif +RAPIDJSON_NAMESPACE_BEGIN +template struct STATIC_ASSERTION_FAILURE; +template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; +template struct StaticAssertTest {}; +RAPIDJSON_NAMESPACE_END + +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y + +#if defined(__GNUC__) +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) +#else +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif +#ifndef __clang__ +//!@endcond +#endif + +/*! \def RAPIDJSON_STATIC_ASSERT + \brief (Internal) macro to check for conditions at compile-time + \param x compile-time condition + \hideinitializer + */ +#define RAPIDJSON_STATIC_ASSERT(x) \ + typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ + sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ + RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#endif + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY + +//! Compiler branching hint for expression with high probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression likely to be true. +*/ +#ifndef RAPIDJSON_LIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define RAPIDJSON_LIKELY(x) (x) +#endif +#endif + +//! Compiler branching hint for expression with low probability to be true. +/*! + \ingroup RAPIDJSON_CONFIG + \param x Boolean expression unlikely to be true. +*/ +#ifndef RAPIDJSON_UNLIKELY +#if defined(__GNUC__) || defined(__clang__) +#define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define RAPIDJSON_UNLIKELY(x) (x) +#endif +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Helpers + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN + +#define RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define RAPIDJSON_MULTILINEMACRO_END \ +} while((void)0, 0) + +// adopted from Boost +#define RAPIDJSON_VERSION_CODE(x,y,z) \ + (((x)*100000) + ((y)*100) + (z)) + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF + +#if defined(__GNUC__) +#define RAPIDJSON_GNUC \ + RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) +#endif + +#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,2,0)) + +#define RAPIDJSON_PRAGMA(x) _Pragma(RAPIDJSON_STRINGIFY(x)) +#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(GCC diagnostic x) +#define RAPIDJSON_DIAG_OFF(x) \ + RAPIDJSON_DIAG_PRAGMA(ignored RAPIDJSON_STRINGIFY(RAPIDJSON_JOIN(-W,x))) + +// push/pop support in Clang and GCC>=4.6 +#if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) +#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) +#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) +#else // GCC >= 4.2, < 4.6 +#define RAPIDJSON_DIAG_PUSH /* ignored */ +#define RAPIDJSON_DIAG_POP /* ignored */ +#endif + +#elif defined(_MSC_VER) + +// pragma (MSVC specific) +#define RAPIDJSON_PRAGMA(x) __pragma(x) +#define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(warning(x)) + +#define RAPIDJSON_DIAG_OFF(x) RAPIDJSON_DIAG_PRAGMA(disable: x) +#define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) +#define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) + +#else + +#define RAPIDJSON_DIAG_OFF(x) /* ignored */ +#define RAPIDJSON_DIAG_PUSH /* ignored */ +#define RAPIDJSON_DIAG_POP /* ignored */ + +#endif // RAPIDJSON_DIAG_* + +/////////////////////////////////////////////////////////////////////////////// +// C++11 features + +#ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS +#if defined(__clang__) +#if __has_feature(cxx_rvalue_references) && \ + (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1600) + +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#else +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS + +#ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) +// (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#else +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 +#endif +#endif +#if RAPIDJSON_HAS_CXX11_NOEXCEPT +#define RAPIDJSON_NOEXCEPT noexcept +#else +#define RAPIDJSON_NOEXCEPT /* noexcept */ +#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT + +// no automatic detection, yet +#ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS +#define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 +#endif + +#ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR +#if defined(__clang__) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700) +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 +#else +#define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 +#endif +#endif // RAPIDJSON_HAS_CXX11_RANGE_FOR + +//!@endcond + +/////////////////////////////////////////////////////////////////////////////// +// new/delete + +#ifndef RAPIDJSON_NEW +///! customization point for global \c new +#define RAPIDJSON_NEW(x) new x +#endif +#ifndef RAPIDJSON_DELETE +///! customization point for global \c delete +#define RAPIDJSON_DELETE(x) delete x +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Type + +/*! \namespace rapidjson + \brief main RapidJSON namespace + \see RAPIDJSON_NAMESPACE +*/ +RAPIDJSON_NAMESPACE_BEGIN + +//! Type of JSON value +enum Type { + kNullType = 0, //!< null + kFalseType = 1, //!< false + kTrueType = 2, //!< true + kObjectType = 3, //!< object + kArrayType = 4, //!< array + kStringType = 5, //!< string + kNumberType = 6 //!< number +}; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/reader.h b/ext/librethinkdbxx/src/rapidjson/reader.h new file mode 100644 index 00000000..19f8849b --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/reader.h @@ -0,0 +1,1879 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_READER_H_ +#define RAPIDJSON_READER_H_ + +/*! \file reader.h */ + +#include "allocators.h" +#include "stream.h" +#include "encodedstream.h" +#include "internal/meta.h" +#include "internal/stack.h" +#include "internal/strtod.h" +#include + +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4702) // unreachable code +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(old-style-cast) +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define RAPIDJSON_NOTHING /* deliberately empty */ +#ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN +#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ + RAPIDJSON_MULTILINEMACRO_END +#endif +#define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) +//!@endcond + +/*! \def RAPIDJSON_PARSE_ERROR_NORETURN + \ingroup RAPIDJSON_ERRORS + \brief Macro to indicate a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + This macros can be used as a customization point for the internal + error handling mechanism of RapidJSON. + + A common usage model is to throw an exception instead of requiring the + caller to explicitly check the \ref rapidjson::GenericReader::Parse's + return value: + + \code + #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ + throw ParseException(parseErrorCode, #parseErrorCode, offset) + + #include // std::runtime_error + #include "rapidjson/error/error.h" // rapidjson::ParseResult + + struct ParseException : std::runtime_error, rapidjson::ParseResult { + ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) + : std::runtime_error(msg), ParseResult(code, offset) {} + }; + + #include "rapidjson/reader.h" + \endcode + + \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse + */ +#ifndef RAPIDJSON_PARSE_ERROR_NORETURN +#define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ + SetParseError(parseErrorCode, offset); \ + RAPIDJSON_MULTILINEMACRO_END +#endif + +/*! \def RAPIDJSON_PARSE_ERROR + \ingroup RAPIDJSON_ERRORS + \brief (Internal) macro to indicate and handle a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) + + Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. + + \see RAPIDJSON_PARSE_ERROR_NORETURN + \hideinitializer + */ +#ifndef RAPIDJSON_PARSE_ERROR +#define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ + RAPIDJSON_MULTILINEMACRO_BEGIN \ + RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ + RAPIDJSON_MULTILINEMACRO_END +#endif + +#include "error/error.h" // ParseErrorCode, ParseResult + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// ParseFlag + +/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kParseDefaultFlags definition. + + User can define this as any \c ParseFlag combinations. +*/ +#ifndef RAPIDJSON_PARSE_DEFAULT_FLAGS +#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags +#endif + +//! Combination of parseFlags +/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream + */ +enum ParseFlag { + kParseNoFlags = 0, //!< No flags are set. + kParseInsituFlag = 1, //!< In-situ(destructive) parsing. + kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. + kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. + kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. + kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. + kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. + kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS +}; + +/////////////////////////////////////////////////////////////////////////////// +// Handler + +/*! \class rapidjson::Handler + \brief Concept for receiving events from GenericReader upon parsing. + The functions return true if no error occurs. If they return false, + the event publisher should terminate the process. +\code +concept Handler { + typename Ch; + + bool Null(); + bool Bool(bool b); + bool Int(int i); + bool Uint(unsigned i); + bool Int64(int64_t i); + bool Uint64(uint64_t i); + bool Double(double d); + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType length, bool copy); + bool String(const Ch* str, SizeType length, bool copy); + bool StartObject(); + bool Key(const Ch* str, SizeType length, bool copy); + bool EndObject(SizeType memberCount); + bool StartArray(); + bool EndArray(SizeType elementCount); +}; +\endcode +*/ +/////////////////////////////////////////////////////////////////////////////// +// BaseReaderHandler + +//! Default implementation of Handler. +/*! This can be used as base class of any reader handler. + \note implements Handler concept +*/ +template, typename Derived = void> +struct BaseReaderHandler { + typedef typename Encoding::Ch Ch; + + typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; + + bool Default() { return true; } + bool Null() { return static_cast(*this).Default(); } + bool Bool(bool) { return static_cast(*this).Default(); } + bool Int(int) { return static_cast(*this).Default(); } + bool Uint(unsigned) { return static_cast(*this).Default(); } + bool Int64(int64_t) { return static_cast(*this).Default(); } + bool Uint64(uint64_t) { return static_cast(*this).Default(); } + bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool StartObject() { return static_cast(*this).Default(); } + bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool EndObject(SizeType) { return static_cast(*this).Default(); } + bool StartArray() { return static_cast(*this).Default(); } + bool EndArray(SizeType) { return static_cast(*this).Default(); } +}; + +/////////////////////////////////////////////////////////////////////////////// +// StreamLocalCopy + +namespace internal { + +template::copyOptimization> +class StreamLocalCopy; + +//! Do copy optimization. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original), original_(original) {} + ~StreamLocalCopy() { original_ = s; } + + Stream s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; + + Stream& original_; +}; + +//! Keep reference. +template +class StreamLocalCopy { +public: + StreamLocalCopy(Stream& original) : s(original) {} + + Stream& s; + +private: + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// SkipWhitespace + +//! Skip the JSON white spaces in a stream. +/*! \param is A input stream for skipping white spaces. + \note This function has SSE2/SSE4.2 specialization. +*/ +template +void SkipWhitespace(InputStream& is) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + typename InputStream::Ch c; + while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') + s.Take(); +} + +inline const char* SkipWhitespace(const char* p, const char* end) { + while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + return p; +} + +#ifdef RAPIDJSON_SSE42 +//! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) { // some of characters is non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The middle of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) { // some of characters is non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#elif defined(RAPIDJSON_SSE2) + +//! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The rest of string + #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; + #undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (; p <= end - 16; p += 16) { + const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) { // some of characters may be non-whitespace +#ifdef _MSC_VER // Find the index of first non-whitespace + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; +#else + return p + __builtin_ffs(r) - 1; +#endif + } + } + + return SkipWhitespace(p, end); +} + +#endif // RAPIDJSON_SSE2 + +#ifdef RAPIDJSON_SIMD +//! Template function specialization for InsituStringStream +template<> inline void SkipWhitespace(InsituStringStream& is) { + is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); +} + +//! Template function specialization for StringStream +template<> inline void SkipWhitespace(StringStream& is) { + is.src_ = SkipWhitespace_SIMD(is.src_); +} + +template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { + is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); +} +#endif // RAPIDJSON_SIMD + +/////////////////////////////////////////////////////////////////////////////// +// GenericReader + +//! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. +/*! GenericReader parses JSON text from a stream, and send events synchronously to an + object implementing Handler concept. + + It needs to allocate a stack for storing a single decoded string during + non-destructive parsing. + + For in-situ parsing, the decoded string is directly written to the source + text string, no temporary buffer is required. + + A GenericReader object can be reused for parsing multiple JSON text. + + \tparam SourceEncoding Encoding of the input stream. + \tparam TargetEncoding Encoding of the parse output. + \tparam StackAllocator Allocator type for stack. +*/ +template +class GenericReader { +public: + typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type + + //! Constructor. + /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) + \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) + */ + GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} + + //! Parse JSON text. + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + if (parseFlags & kParseIterativeFlag) + return IterativeParse(is, handler); + + parseResult_.Clear(); + + ClearStackOnExit scope(*this); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + else { + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (!(parseFlags & kParseStopWhenDoneFlag)) { + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + } + } + + return parseResult_; + } + + //! Parse JSON text (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) { + return Parse(is, handler); + } + + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } + + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } + + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } + +protected: + void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } + +private: + // Prohibit copy constructor & assignment operator. + GenericReader(const GenericReader&); + GenericReader& operator=(const GenericReader&); + + void ClearStack() { stack_.Clear(); } + + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit { + explicit ClearStackOnExit(GenericReader& r) : r_(r) {} + ~ClearStackOnExit() { r_.ClearStack(); } + private: + GenericReader& r_; + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + }; + + template + void SkipWhitespaceAndComments(InputStream& is) { + SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) { + while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { + if (Consume(is, '*')) { + while (true) { + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + else if (Consume(is, '*')) { + if (Consume(is, '/')) + break; + } + else + is.Take(); + } + } + else if (RAPIDJSON_LIKELY(Consume(is, '/'))) + while (is.Peek() != '\0' && is.Take() != '\n'); + else + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + SkipWhitespace(is); + } + } + } + + // Parse object: { string : value, ... } + template + void ParseObject(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == '{'); + is.Take(); // Skip '{' + + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, '}')) { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType memberCount = 0;;) { + if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + + ParseString(is, handler, true); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++memberCount; + + switch (is.Peek()) { + case ',': + is.Take(); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; + case '}': + is.Take(); + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + default: + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy + } + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == '}') { + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } + } + } + } + + // Parse array: [ value, ... ] + template + void ParseArray(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == '['); + is.Take(); // Skip '[' + + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + + for (SizeType elementCount = 0;;) { + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + ++elementCount; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + + if (Consume(is, ',')) { + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + } + else if (Consume(is, ']')) { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + + if (parseFlags & kParseTrailingCommasFlag) { + if (is.Peek() == ']') { + if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; + } + } + } + } + + template + void ParseNull(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 'n'); + is.Take(); + + if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { + if (RAPIDJSON_UNLIKELY(!handler.Null())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + void ParseTrue(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 't'); + is.Take(); + + if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + void ParseFalse(InputStream& is, Handler& handler) { + RAPIDJSON_ASSERT(is.Peek() == 'f'); + is.Take(); + + if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { + if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { + if (RAPIDJSON_LIKELY(is.Peek() == expect)) { + is.Take(); + return true; + } + else + return false; + } + + // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). + template + unsigned ParseHex4(InputStream& is, size_t escapeOffset) { + unsigned codepoint = 0; + for (int i = 0; i < 4; i++) { + Ch c = is.Peek(); + codepoint <<= 4; + codepoint += static_cast(c); + if (c >= '0' && c <= '9') + codepoint -= '0'; + else if (c >= 'A' && c <= 'F') + codepoint -= 'A' - 10; + else if (c >= 'a' && c <= 'f') + codepoint -= 'a' - 10; + else { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); + } + is.Take(); + } + return codepoint; + } + + template + class StackStream { + public: + typedef CharType Ch; + + StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} + RAPIDJSON_FORCEINLINE void Put(Ch c) { + *stack_.template Push() = c; + ++length_; + } + + RAPIDJSON_FORCEINLINE void* Push(SizeType count) { + length_ += count; + return stack_.template Push(count); + } + + size_t Length() const { return length_; } + + Ch* Pop() { + return stack_.template Pop(length_); + } + + private: + StackStream(const StackStream&); + StackStream& operator=(const StackStream&); + + internal::Stack& stack_; + SizeType length_; + }; + + // Parse string and generate String event. Different code paths for kParseInsituFlag. + template + void ParseString(InputStream& is, Handler& handler, bool isKey = false) { + internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); + + RAPIDJSON_ASSERT(s.Peek() == '\"'); + s.Take(); // Skip '\"' + + bool success = false; + if (parseFlags & kParseInsituFlag) { + typename InputStream::Ch *head = s.PutBegin(); + ParseStringToStream(s, s); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + size_t length = s.PutEnd(head) - 1; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); + } + else { + StackStream stackStream(stack_); + ParseStringToStream(s, stackStream); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SizeType length = static_cast(stackStream.Length()) - 1; + const typename TargetEncoding::Ch* const str = stackStream.Pop(); + success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); + } + if (RAPIDJSON_UNLIKELY(!success)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + } + + // Parse string to an output is + // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. + template + RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + static const char escape[256] = { + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 + }; +#undef Z16 +//!@endcond + + for (;;) { + // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + + Ch c = is.Peek(); + if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape + size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset + is.Take(); + Ch e = is.Peek(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { + is.Take(); + os.Put(static_cast(escape[static_cast(e)])); + } + else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode + is.Take(); + unsigned codepoint = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { + // Handle UTF-16 surrogate pair + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } + TEncoding::Encode(os, codepoint); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); + } + else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote + is.Take(); + os.Put('\0'); // null-terminate the string + return; + } + else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); + } + else { + size_t offset = is.Tell(); + if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); + } + } + } + + template + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { + // Do nothing for generic version + } + +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType length; + #ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; + #else + length = static_cast(__builtin_ffs(r) - 1); + #endif + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16, q += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + for (const char* pend = p + length; p != pend; ) + *q++ = *p++; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + p += length; + break; + } + } + + is.src_ = is.dst_ = p; + } +#endif + + template + class NumberStream; + + template + class NumberStream { + public: + typedef typename InputStream::Ch Ch; + + NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } + RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } + RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char) {} + + size_t Tell() { return is.Tell(); } + size_t Length() { return 0; } + const char* Pop() { return 0; } + + protected: + NumberStream& operator=(const NumberStream&); + + InputStream& is; + }; + + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch TakePush() { + stackStream.Put(static_cast(Base::is.Peek())); + return Base::is.Take(); + } + + RAPIDJSON_FORCEINLINE void Push(char c) { + stackStream.Put(c); + } + + size_t Length() { return stackStream.Length(); } + + const char* Pop() { + stackStream.Put('\0'); + return stackStream.Pop(); + } + + private: + StackStream stackStream; + }; + + template + class NumberStream : public NumberStream { + typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } + }; + + template + void ParseNumber(InputStream& is, Handler& handler) { + internal::StreamLocalCopy copy(is); + NumberStream s(*this, copy.s); + + size_t startOffset = s.Tell(); + double d = 0.0; + bool useNanOrInf = false; + + // Parse minus + bool minus = Consume(s, '-'); + + // Parse int: zero / ( digit1-9 *DIGIT ) + unsigned i = 0; + uint64_t i64 = 0; + bool use64bit = false; + int significandDigit = 0; + if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { + i = 0; + s.TakePush(); + } + else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { + i = static_cast(s.TakePush() - '0'); + + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 + if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 + if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { + i64 = i; + use64bit = true; + break; + } + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + } + // Parse NaN or Infinity here + else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { + useNanOrInf = true; + if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) { + d = std::numeric_limits::quiet_NaN(); + } + else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) { + d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + + // Parse 64bit int + bool useDouble = false; + if (use64bit) { + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { + d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { + d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + } + + // Force double for big integer + if (useDouble) { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + d = d * 10 + (s.TakePush() - '0'); + } + } + + // Parse frac = decimal-point 1*DIGIT + int expFrac = 0; + size_t decimalPosition; + if (Consume(s, '.')) { + decimalPosition = s.Length(); + + if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); + + if (!useDouble) { +#if RAPIDJSON_64BIT + // Use i64 to store significand in 64-bit architecture + if (!use64bit) + i64 = i; + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path + break; + else { + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + --expFrac; + if (i64 != 0) + significandDigit++; + } + } + + d = static_cast(i64); +#else + // Use double to store significand in 32-bit architecture + d = static_cast(use64bit ? i64 : i); +#endif + useDouble = true; + } + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + if (significandDigit < 17) { + d = d * 10.0 + (s.TakePush() - '0'); + --expFrac; + if (RAPIDJSON_LIKELY(d > 0.0)) + significandDigit++; + } + else + s.TakePush(); + } + } + else + decimalPosition = s.Length(); // decimal position at the end of integer. + + // Parse exp = e [ minus / plus ] 1*DIGIT + int exp = 0; + if (Consume(s, 'e') || Consume(s, 'E')) { + if (!useDouble) { + d = static_cast(use64bit ? i64 : i); + useDouble = true; + } + + bool expMinus = false; + if (Consume(s, '+')) + ; + else if (Consume(s, '-')) + expMinus = true; + + if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = static_cast(s.Take() - '0'); + if (expMinus) { + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (exp >= 214748364) { // Issue #313: prevent overflow exponent + while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent + s.Take(); + } + } + } + else { // positive exp + int maxExp = 308 - expFrac; + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { + exp = exp * 10 + static_cast(s.Take() - '0'); + if (RAPIDJSON_UNLIKELY(exp > maxExp)) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + } + } + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); + + if (expMinus) + exp = -exp; + } + + // Finish parsing, call event according to the type of number. + bool cont = true; + + if (parseFlags & kParseNumbersAsStringsFlag) { + if (parseFlags & kParseInsituFlag) { + s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch* head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + // unable to insert the \0 character here, it will erase the comma after this number + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); + } + else { + SizeType numCharsToCopy = static_cast(s.Length()); + StringStream srcStream(s.Pop()); + StackStream dstStream(stack_); + while (numCharsToCopy--) { + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + } + dstStream.Put('\0'); + const typename TargetEncoding::Ch* str = dstStream.Pop(); + const SizeType length = static_cast(dstStream.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); + } + } + else { + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); + } + else if (useNanOrInf) { + cont = handler.Double(d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); + } + } + } + if (RAPIDJSON_UNLIKELY(!cont)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); + } + + // Parse any JSON value + template + void ParseValue(InputStream& is, Handler& handler) { + switch (is.Peek()) { + case 'n': ParseNull (is, handler); break; + case 't': ParseTrue (is, handler); break; + case 'f': ParseFalse (is, handler); break; + case '"': ParseString(is, handler); break; + case '{': ParseObject(is, handler); break; + case '[': ParseArray (is, handler); break; + default : + ParseNumber(is, handler); + break; + + } + } + + // Iterative Parsing + + // States + enum IterativeParsingState { + IterativeParsingStartState = 0, + IterativeParsingFinishState, + IterativeParsingErrorState, + + // Object states + IterativeParsingObjectInitialState, + IterativeParsingMemberKeyState, + IterativeParsingKeyValueDelimiterState, + IterativeParsingMemberValueState, + IterativeParsingMemberDelimiterState, + IterativeParsingObjectFinishState, + + // Array states + IterativeParsingArrayInitialState, + IterativeParsingElementState, + IterativeParsingElementDelimiterState, + IterativeParsingArrayFinishState, + + // Single value state + IterativeParsingValueState + }; + + enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; + + // Tokens + enum Token { + LeftBracketToken = 0, + RightBracketToken, + + LeftCurlyBracketToken, + RightCurlyBracketToken, + + CommaToken, + ColonToken, + + StringToken, + FalseToken, + TrueToken, + NullToken, + NumberToken, + + kTokenCount + }; + + RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN +#define N NumberToken +#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N + // Maps from ASCII to Token + static const unsigned char tokenMap[256] = { + N16, // 00~0F + N16, // 10~1F + N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F + N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F + N16, // 40~4F + N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F + N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F + N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F + N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF + }; +#undef N +#undef N16 +//!@endcond + + if (sizeof(Ch) == 1 || static_cast(c) < 256) + return static_cast(tokenMap[static_cast(c)]); + else + return NumberToken; + } + + RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { + // current state x one lookahead token -> new state + static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Start + { + IterativeParsingArrayInitialState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingValueState, // String + IterativeParsingValueState, // False + IterativeParsingValueState, // True + IterativeParsingValueState, // Null + IterativeParsingValueState // Number + }, + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ObjectInitial + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberKey + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingKeyValueDelimiterState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, + // MemberValue + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingMemberDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberDelimiter + { + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ObjectFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ArrayInitial + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // Element + { + IterativeParsingErrorState, // Left bracket + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingElementDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ElementDelimiter + { + IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // ArrayFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Single Value (sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + } + }; // End of G + + return static_cast(G[state][token]); + } + + // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). + // May return a new state on state pop. + template + RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { + (void)token; + + switch (dst) { + case IterativeParsingErrorState: + return dst; + + case IterativeParsingObjectInitialState: + case IterativeParsingArrayInitialState: + { + // Push the state(Element or MemeberValue) if we are nested in another array or value of member. + // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. + IterativeParsingState n = src; + if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) + n = IterativeParsingElementState; + else if (src == IterativeParsingKeyValueDelimiterState) + n = IterativeParsingMemberValueState; + // Push current state. + *stack_.template Push(1) = n; + // Initialize and push the member/element count. + *stack_.template Push(1) = 0; + // Call handler + bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return dst; + } + } + + case IterativeParsingMemberKeyState: + ParseString(is, handler, true); + if (HasParseError()) + return IterativeParsingErrorState; + else + return dst; + + case IterativeParsingKeyValueDelimiterState: + RAPIDJSON_ASSERT(token == ColonToken); + is.Take(); + return dst; + + case IterativeParsingMemberValueState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingElementState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return dst; + + case IterativeParsingMemberDelimiterState: + case IterativeParsingElementDelimiterState: + is.Take(); + // Update member/element count. + *stack_.template Top() = *stack_.template Top() + 1; + return dst; + + case IterativeParsingObjectFinishState: + { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); + return IterativeParsingErrorState; + } + // Get member count. + SizeType c = *stack_.template Pop(1); + // If the object is not empty, count the last member. + if (src == IterativeParsingMemberValueState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndObject(c); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + case IterativeParsingArrayFinishState: + { + // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); + return IterativeParsingErrorState; + } + // Get element count. + SizeType c = *stack_.template Pop(1); + // If the array is not empty, count the last element. + if (src == IterativeParsingElementState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndArray(c); + // On handler short circuits the parsing. + if (!hr) { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else { + is.Take(); + return n; + } + } + + default: + // This branch is for IterativeParsingValueState actually. + // Use `default:` rather than + // `case IterativeParsingValueState:` is for code coverage. + + // The IterativeParsingStartState is not enumerated in this switch-case. + // It is impossible for that case. And it can be caught by following assertion. + + // The IterativeParsingFinishState is not enumerated in this switch-case either. + // It is a "derivative" state which cannot triggered from Predict() directly. + // Therefore it cannot happen here. And it can be caught by following assertion. + RAPIDJSON_ASSERT(dst == IterativeParsingValueState); + + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) { + return IterativeParsingErrorState; + } + return IterativeParsingFinishState; + } + } + + template + void HandleError(IterativeParsingState src, InputStream& is) { + if (HasParseError()) { + // Error flag has been set. + return; + } + + switch (src) { + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; + case IterativeParsingObjectInitialState: + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; + case IterativeParsingKeyValueDelimiterState: + case IterativeParsingArrayInitialState: + case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; + default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; + } + } + + template + ParseResult IterativeParse(InputStream& is, Handler& handler) { + parseResult_.Clear(); + ClearStackOnExit scope(*this); + IterativeParsingState state = IterativeParsingStartState; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + while (is.Peek() != '\0') { + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state, t); + IterativeParsingState d = Transit(state, t, n, is, handler); + + if (d == IterativeParsingErrorState) { + HandleError(state, is); + break; + } + + state = d; + + // Do not further consume streams if a root JSON has been parsed. + if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) + break; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + + // Handle the end of file. + if (state != IterativeParsingFinishState) + HandleError(state, is); + + return parseResult_; + } + + static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. + internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. + ParseResult parseResult_; +}; // class GenericReader + +//! Reader with UTF8 encoding and default allocator. +typedef GenericReader, UTF8<> > Reader; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_READER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/schema.h b/ext/librethinkdbxx/src/rapidjson/schema.h new file mode 100644 index 00000000..b182aa27 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/schema.h @@ -0,0 +1,2006 @@ +// Tencent is pleased to support the open source community by making RapidJSON available-> +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License-> You may obtain a copy of the License at +// +// http://opensource->org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// specific language governing permissions and limitations under the License-> + +#ifndef RAPIDJSON_SCHEMA_H_ +#define RAPIDJSON_SCHEMA_H_ + +#include "document.h" +#include "pointer.h" +#include // abs, floor + +#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 +#endif + +#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX +#include "internal/regex.h" +#elif RAPIDJSON_SCHEMA_USE_STDREGEX +#include +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX +#define RAPIDJSON_SCHEMA_HAS_REGEX 1 +#else +#define RAPIDJSON_SCHEMA_HAS_REGEX 0 +#endif + +#ifndef RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_VERBOSE 0 +#endif + +#if RAPIDJSON_SCHEMA_VERBOSE +#include "stringbuffer.h" +#endif + +RAPIDJSON_DIAG_PUSH + +#if defined(__GNUC__) +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(weak-vtables) +RAPIDJSON_DIAG_OFF(exit-time-destructors) +RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) +RAPIDJSON_DIAG_OFF(variadic-macros) +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Verbose Utilities + +#if RAPIDJSON_SCHEMA_VERBOSE + +namespace internal { + +inline void PrintInvalidKeyword(const char* keyword) { + printf("Fail keyword: %s\n", keyword); +} + +inline void PrintInvalidKeyword(const wchar_t* keyword) { + wprintf(L"Fail keyword: %ls\n", keyword); +} + +inline void PrintInvalidDocument(const char* document) { + printf("Fail document: %s\n\n", document); +} + +inline void PrintInvalidDocument(const wchar_t* document) { + wprintf(L"Fail document: %ls\n\n", document); +} + +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { + printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +} + +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { + wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +} // namespace internal + +#endif // RAPIDJSON_SCHEMA_VERBOSE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_INVALID_KEYWORD_RETURN + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) +#else +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) +#endif + +#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + context.invalidKeyword = keyword.GetString();\ + RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ + return false;\ +RAPIDJSON_MULTILINEMACRO_END + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations + +template +class GenericSchemaDocument; + +namespace internal { + +template +class Schema; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaValidator + +class ISchemaValidator { +public: + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaStateFactory + +template +class ISchemaStateFactory { +public: + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Hasher + +// For comparison of compound value +template +class Hasher { +public: + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) { + Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) { + WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) { + uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) { + uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const { + RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } + +private: + static const size_t kDefaultSize = 256; + struct Number { + union U { + uint64_t u; + int64_t i; + }u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) { + // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) { + static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidationContext + +template +struct SchemaValidationContext { + typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType { + kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : + factory(f), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() { + if (hasher) + factory.DestroryHasher(hasher); + if (validators) { + for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) { + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Schema + +template +class Schema { +public: + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false) + { + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) { + type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) { + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; + char buffer[256 + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); + } + } + + if (schemaDocument) { + AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } + + if (const ValueType* v = GetMember(value, GetNotString())) { + schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } + + // Object + + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { + // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); + + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); + + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); + } + + if (allProperties.Size() > 0) { + propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) { + new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = GetTypeless(); + } + } + } + + if (properties && properties->IsObject()) { + PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { + SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { + PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { + new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) { + SizeType index; + if (FindPropertyIndex(*itr, &index)) { + properties_[index].required = true; + hasRequired_ = true; + } + } + + if (dependencies && dependencies->IsObject()) { + PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) { + if (itr->value.IsArray()) { + properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { + SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; + } + } + else if (itr->value.IsObject()) { + hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } + } + } + } + + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { + if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) { + PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) { // Tuple validation + itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } + } + + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); + + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { + if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); + } + + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); + + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); + + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); + + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); + + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); + + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); + + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + } + + ~Schema() { + if (allocator_) { + allocator_->Free(enum_); + } + if (properties_) { + for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); + } + if (patternProperties_) { + for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); + } + AllocatorType::Free(itemsTuple_); +#if RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) { + pattern_->~RegexType(); + allocator_->Free(pattern_); + } +#endif + } + + bool BeginValue(Context& context) const { + if (context.inArray) { + if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) { + if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = GetTypeless(); + else + RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + else + context.valueSchema = GetTypeless(); + + context.arrayElementIndex++; + } + return true; + } + + RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + if (context.patternPropertiesValidatorCount > 0) { + bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) { + patternValid = false; + break; + } + + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { + if (!patternValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { + if (!patternValid || !otherValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + + if (enum_) { + const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); + foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + + if (anyOf_.schemas) { + for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); + foundAny:; + } + + if (oneOf_.schemas) { + bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) { + if (oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + else + oneValid = true; + } + if (!oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } + + if (not_ && context.validators[notValidatorIndex_]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + + return true; + } + + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Int(Context& context, int i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint(Context& context, unsigned u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint64(Context& context, uint64_t u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Double(Context& context, double d) const { + if (!(type_ & (1 << kNumberSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; + + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; + + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; + + return CreateParallelValidator(context); + } + + bool String(Context& context, const Ch* str, SizeType length, bool) const { + if (!(type_ & (1 << kStringSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) { + SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) { + if (count < minLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + if (count > maxLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (hasDependencies_ || hasRequired_) { + context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) { // pre-allocate schema array + SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const { + if (patternProperties_) { + context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { + if (context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = GetTypeless(); + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) { + if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = GetTypeless(); + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) { + context.valueSchema = GetTypeless(); + return true; + } + + if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + + return true; + } + + bool EndObject(Context& context, SizeType memberCount) const { + if (hasRequired_) + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required) + if (!context.propertyExist[index]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + + if (memberCount < minProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + + if (memberCount > maxProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + + if (hasDependencies_) { + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) + if (context.propertyExist[sourceIndex]) { + if (properties_[sourceIndex].dependencies) { + for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + else if (properties_[sourceIndex].dependenciesSchema) + if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + } + + return true; + } + + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + context.arrayElementIndex = 0; + context.inArray = true; + + return CreateParallelValidator(context); + } + + bool EndArray(Context& context, SizeType elementCount) const { + context.inArray = false; + + if (elementCount < minItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + + if (elementCount > maxItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + + return true; + } + + // Generate functions for string literal according to Ch +#define RAPIDJSON_STRING_(name, ...) \ + static const ValueType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const ValueType v(s, sizeof(s) / sizeof(Ch) - 1);\ + return v;\ + } + + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + RAPIDJSON_STRING_(Not, 'n', 'o', 't') + RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + +#undef RAPIDJSON_STRING_ + +private: + enum SchemaValueType { + kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + typedef internal::GenericRegex RegexType; +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + typedef std::basic_regex RegexType; +#else + typedef char RegexType; +#endif + + struct SchemaArray { + SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + static const SchemaType* GetTypeless() { + static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); + return &typeless; + } + + template + void AddUniqueElement(V1& a, const V2& v) { + for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) { + typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { + if (const ValueType* v = GetMember(value, name)) { + if (v->IsArray() && v->Size() > 0) { + PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); + if (!r->IsValid()) { + r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { + return pattern->Search(str); + } +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) + try { + return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) { + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { + std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } +#else + template + RegexType* CreatePattern(const ValueType&) { return 0; } + + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } +#endif // RAPIDJSON_SCHEMA_USE_STDREGEX + + void AddType(const ValueType& type) { + if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const { + if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) { + RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) { + for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { + for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { + SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { + *outIndex = index; + return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsInt64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsUint64()) { + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsInt64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsUint64()) + /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckUint(Context& context, uint64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) { + if (minimum_.IsUint64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsUint64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (i % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const { + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const { + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const { + double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + return true; + } + + struct Property { + Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty { + PatternProperty() : schema(), pattern() {} + ~PatternProperty() { + if (pattern) { + pattern->~RegexType(); + AllocatorType::Free(pattern); + } + } + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; +}; + +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = buffer[i]; + } +}; + +// Partial specialized version for char to prevent buffer copying. +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + if (sizeof(SizeType) == 4) { + char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else { + char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// IGenericRemoteSchemaDocumentProvider + +template +class IGenericRemoteSchemaDocumentProvider { +public: + typedef typename SchemaDocumentType::Ch Ch; + + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaDocument + +//! JSON schema document. +/*! + A JSON schema document is a compiled version of a JSON schema. + It is basically a tree of internal::Schema. + + \note This is an immutable class (i.e. its instance cannot be modified after construction). + \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. + \tparam Allocator Allocator type for allocating memory of this document. +*/ +template +class GenericSchemaDocument { +public: + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) { + SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) { + if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) { + new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); + } + } + refEntry->~SchemaRefEntry(); + } + + RAPIDJSON_ASSERT(root_ != 0); + + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)) + { + rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + } +#endif + + //! Destructor + ~GenericSchemaDocument() { + while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); + + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } + +private: + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry { + SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry { + SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() { + if (owned) { + schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + if (schema) + *schema = SchemaType::GetTypeless(); + + if (v.GetType() == kObjectType) { + const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) { + if (!HandleRefSchema(pointer, schema, v, document)) { + SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); + if (schema) + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { + static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) { + SizeType len = itr->value.GetStringLength(); + if (len > 0) { + const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) { // Remote reference, resolve immediately + if (remoteProvider_) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) { + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { + if (schema) + *schema = sc; + return true; + } + } + } + } + } + else if (s[i] == '#') { // Local reference, defer resolution + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; + + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref +}; + +//! GenericSchemaDocument using Value type. +typedef GenericSchemaDocument SchemaDocument; +//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaValidator + +//! JSON Schema Validator. +/*! + A SAX style JSON schema validator. + It uses a \c GenericSchemaDocument to validate SAX events. + It delegates the incoming SAX events to an output handler. + The default output handler does nothing. + It can be reused multiple times by calling \c Reset(). + + \tparam SchemaDocumentType Type of schema document. + \tparam OutputHandler Type of output handler. Default handler does nothing. + \tparam StateAllocator Allocator for storing the internal validation states. +*/ +template < + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> +class GenericSchemaValidator : + public internal::ISchemaStateFactory, + public internal::ISchemaValidator +{ +public: + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + outputHandler_(GetNullHandler()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + outputHandler_(outputHandler), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Destructor. + ~GenericSchemaValidator() { + Reset(); + RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() { + while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const { + return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const { + return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const { + return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + internal::PrintInvalidDocument(documentStack_.template Bottom());\ +RAPIDJSON_MULTILINEMACRO_END +#else +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() +#endif + +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ + if (!valid_) return false; \ + if (!BeginValue() || !CurrentSchema().method arg1) {\ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ + return valid_ = false;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ + for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ + if (context->hasher)\ + static_cast(context->hasher)->method arg2;\ + if (context->validators)\ + for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ + static_cast(context->validators[i_])->method arg2;\ + if (context->patternPropertiesValidators)\ + for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ + static_cast(context->patternPropertiesValidators[i_])->method arg2;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ + return valid_ = EndValue() && outputHandler_.method arg2 + +#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ + RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) + + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } + bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = outputHandler_.StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) { + if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = outputHandler_.Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = outputHandler_.StartArray(); + } + + bool EndArray(SizeType elementCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } + +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ +#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ +#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ + + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, +#if RAPIDJSON_SCHEMA_VERBOSE + depth_ + 1, +#endif + &GetStateAllocator()); + } + + virtual void DestroySchemaValidator(ISchemaValidator* validator) { + GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } + + virtual void* CreateHasher() { + return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } + + virtual uint64_t GetHashCode(void* hasher) { + return static_cast(hasher)->GetHashCode(); + } + + virtual void DestroryHasher(void* hasher) { + HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } + + virtual void* MallocState(size_t size) { + return GetStateAllocator().Malloc(size); + } + + virtual void FreeState(void* p) { + return StateAllocator::Free(p); + } + +private: + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; + + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth, +#endif + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + outputHandler_(GetNullHandler()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(depth) +#endif + { + } + + StateAllocator& GetStateAllocator() { + if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); + return *stateAllocator_; + } + + bool BeginValue() { + if (schemaStack_.Empty()) + PushSchema(root_); + else { + if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; + + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + if (CurrentContext().valueSchema) + PushSchema(*CurrentContext().valueSchema); + + if (count > 0) { + CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } + + CurrentContext().arrayUniqueness = valueUniqueness; + } + return true; + } + + bool EndValue() { + if (!CurrentSchema().EndValue(CurrentContext())) + return false; + +#if RAPIDJSON_SCHEMA_VERBOSE + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); +#endif + + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) { + Context& context = CurrentContext(); + if (context.valueUniqueness) { + HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) + RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) { + documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) { + if (str[i] == '~') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } + + RAPIDJSON_FORCEINLINE void PopSchema() { + Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { + a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static OutputHandler& GetNullHandler() { + static OutputHandler nullHandler; + return nullHandler; + } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + OutputHandler& outputHandler_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + bool valid_; +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth_; +#endif +}; + +typedef GenericSchemaValidator SchemaValidator; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidatingReader + +//! A helper class for parsing with validation. +/*! + This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). + + \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam SourceEncoding Encoding of the input stream. + \tparam SchemaDocumentType Type of schema document. + \tparam StackAllocator Allocator type for stack. +*/ +template < + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader { +public: + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} + + template + bool operator()(Handler& handler) { + GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) { + invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + } + else { + invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + +private: + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + bool isValid_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_SCHEMA_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/stream.h b/ext/librethinkdbxx/src/rapidjson/stream.h new file mode 100644 index 00000000..fef82c25 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/stream.h @@ -0,0 +1,179 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "rapidjson.h" + +#ifndef RAPIDJSON_STREAM_H_ +#define RAPIDJSON_STREAM_H_ + +#include "encodings.h" + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Stream + +/*! \class rapidjson::Stream + \brief Concept for reading and writing characters. + + For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). + + For write-only stream, only need to implement Put() and Flush(). + +\code +concept Stream { + typename Ch; //!< Character type of the stream. + + //! Read the current character from stream without moving the read cursor. + Ch Peek() const; + + //! Read the current character from stream and moving the read cursor to next character. + Ch Take(); + + //! Get the current read cursor. + //! \return Number of characters read from start. + size_t Tell(); + + //! Begin writing operation at the current read pointer. + //! \return The begin writer pointer. + Ch* PutBegin(); + + //! Write a character. + void Put(Ch c); + + //! Flush the buffer. + void Flush(); + + //! End the writing operation. + //! \param begin The begin write pointer returned by PutBegin(). + //! \return Number of characters written. + size_t PutEnd(Ch* begin); +} +\endcode +*/ + +//! Provides additional information for stream. +/*! + By using traits pattern, this type provides a default configuration for stream. + For custom stream, this type can be specialized for other configuration. + See TEST(Reader, CustomStringStream) in readertest.cpp for example. +*/ +template +struct StreamTraits { + //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; +}; + +//! Reserve n characters for writing to a stream. +template +inline void PutReserve(Stream& stream, size_t count) { + (void)stream; + (void)count; +} + +//! Write character to a stream, presuming buffer is reserved. +template +inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { + stream.Put(c); +} + +//! Put N copies of a character to a stream. +template +inline void PutN(Stream& stream, Ch c, size_t n) { + PutReserve(stream, n); + for (size_t i = 0; i < n; i++) + PutUnsafe(stream, c); +} + +/////////////////////////////////////////////////////////////////////////////// +// StringStream + +//! Read-only string stream. +/*! \note implements Stream concept +*/ +template +struct GenericStringStream { + typedef typename Encoding::Ch Ch; + + GenericStringStream(const Ch *src) : src_(src), head_(src) {} + + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! String stream with UTF8 encoding. +typedef GenericStringStream > StringStream; + +/////////////////////////////////////////////////////////////////////////////// +// InsituStringStream + +//! A read-write string stream. +/*! This string stream is particularly designed for in-situ parsing. + \note implements Stream concept +*/ +template +struct GenericInsituStringStream { + typedef typename Encoding::Ch Ch; + + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } + + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} + + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } + + Ch* src_; + Ch* dst_; + Ch* head_; +}; + +template +struct StreamTraits > { + enum { copyOptimization = 1 }; +}; + +//! Insitu string stream with UTF8 encoding. +typedef GenericInsituStringStream > InsituStringStream; + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_STREAM_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/stringbuffer.h b/ext/librethinkdbxx/src/rapidjson/stringbuffer.h new file mode 100644 index 00000000..78f34d20 --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/stringbuffer.h @@ -0,0 +1,117 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_STRINGBUFFER_H_ +#define RAPIDJSON_STRINGBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + +#include "internal/stack.h" + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output stream. +/*! + \tparam Encoding Encoding of the stream. + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +class GenericStringBuffer { +public: + typedef typename Encoding::Ch Ch; + + GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} + GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { + if (&rhs != this) + stack_ = std::move(rhs.stack_); + return *this; + } +#endif + + void Put(Ch c) { *stack_.template Push() = c; } + void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.ShrinkToFit(); + stack_.template Pop(1); + } + + void Reserve(size_t count) { stack_.template Reserve(count); } + Ch* Push(size_t count) { return stack_.template Push(count); } + Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetString() const { + // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.template Pop(1); + + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; + +private: + // Prohibit copy constructor & assignment operator. + GenericStringBuffer(const GenericStringBuffer&); + GenericStringBuffer& operator=(const GenericStringBuffer&); +}; + +//! String buffer with UTF8 encoding +typedef GenericStringBuffer > StringBuffer; + +template +inline void PutReserve(GenericStringBuffer& stream, size_t count) { + stream.Reserve(count); +} + +template +inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { + stream.PutUnsafe(c); +} + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { + std::memset(stream.stack_.Push(n), c, n * sizeof(c)); +} + +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_STRINGBUFFER_H_ diff --git a/ext/librethinkdbxx/src/rapidjson/writer.h b/ext/librethinkdbxx/src/rapidjson/writer.h new file mode 100644 index 00000000..7d0610eb --- /dev/null +++ b/ext/librethinkdbxx/src/rapidjson/writer.h @@ -0,0 +1,609 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_WRITER_H_ +#define RAPIDJSON_WRITER_H_ + +#include "stream.h" +#include "internal/stack.h" +#include "internal/strfunc.h" +#include "internal/dtoa.h" +#include "internal/itoa.h" +#include "stringbuffer.h" +#include // placement new + +#if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) +#include +#pragma intrinsic(_BitScanForward) +#endif +#ifdef RAPIDJSON_SSE42 +#include +#elif defined(RAPIDJSON_SSE2) +#include +#endif + +#ifdef _MSC_VER +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// WriteFlag + +/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kWriteDefaultFlags definition. + + User can define this as any \c WriteFlag combinations. +*/ +#ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS +#define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags +#endif + +//! Combination of writeFlags +enum WriteFlag { + kWriteNoFlags = 0, //!< No flags are set. + kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. + kWriteNanAndInfFlag = 2, //!< Allow writing of Inf, -Inf and NaN. + kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS +}; + +//! JSON writer +/*! Writer implements the concept Handler. + It generates JSON text by events to an output os. + + User may programmatically calls the functions of a writer to generate JSON text. + + On the other side, a writer can also be passed to objects that generates events, + + for example Reader::Parse() and Document::Accept(). + + \tparam OutputStream Type of output stream. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. + \note implements Handler concept +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class Writer { +public: + typedef typename SourceEncoding::Ch Ch; + + static const int kDefaultMaxDecimalPlaces = 324; + + //! Constructor + /*! \param os Output stream. + \param stackAllocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit + Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + explicit + Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + //! Reset the writer with a new stream. + /*! + This function reset the writer with a new stream and default settings, + in order to make a Writer object reusable for output multiple JSONs. + + \param os New output stream. + \code + Writer writer(os1); + writer.StartObject(); + // ... + writer.EndObject(); + + writer.Reset(os2); + writer.StartObject(); + // ... + writer.EndObject(); + \endcode + */ + void Reset(OutputStream& os) { + os_ = &os; + hasRoot_ = false; + level_stack_.Clear(); + } + + //! Checks whether the output is a complete JSON. + /*! + A complete JSON has a complete root object or array. + */ + bool IsComplete() const { + return hasRoot_ && level_stack_.Empty(); + } + + int GetMaxDecimalPlaces() const { + return maxDecimalPlaces_; + } + + //! Sets the maximum number of decimal places for double output. + /*! + This setting truncates the output with specified number of decimal places. + + For example, + + \code + writer.SetMaxDecimalPlaces(3); + writer.StartArray(); + writer.Double(0.12345); // "0.123" + writer.Double(0.0001); // "0.0" + writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) + writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) + writer.EndArray(); + \endcode + + The default setting does not truncate any decimal places. You can restore to this setting by calling + \code + writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); + \endcode + */ + void SetMaxDecimalPlaces(int maxDecimalPlaces) { + maxDecimalPlaces_ = maxDecimalPlaces; + } + + /*!@name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { Prefix(kNullType); return WriteNull(); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return WriteBool(b); } + bool Int(int i) { Prefix(kNumberType); return WriteInt(i); } + bool Uint(unsigned u) { Prefix(kNumberType); return WriteUint(u); } + bool Int64(int64_t i64) { Prefix(kNumberType); return WriteInt64(i64); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return WriteUint64(u64); } + + //! Writes the given \c double value to the stream + /*! + \param d The value to be written. + \return Whether it is succeed. + */ + bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + Prefix(kNumberType); + return WriteString(str, length); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + Prefix(kStringType); + return WriteString(str, length); + } + +#if RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + Prefix(kObjectType); + new (level_stack_.template Push()) Level(false); + return WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + bool ret = WriteEndObject(); + if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + os_->Flush(); + return ret; + } + + bool StartArray() { + Prefix(kArrayType); + new (level_stack_.template Push()) Level(true); + return WriteStartArray(); + } + + bool EndArray(SizeType elementCount = 0) { + (void)elementCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + bool ret = WriteEndArray(); + if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + os_->Flush(); + return ret; + } + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + */ + bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return WriteRawValue(json, length); } + +protected: + //! Information for each nested level + struct Level { + Level(bool inArray_) : valueCount(0), inArray(inArray_) {} + size_t valueCount; //!< number of values in this level + bool inArray; //!< true if in array, otherwise in object + }; + + static const size_t kDefaultLevelDepth = 32; + + bool WriteNull() { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; + } + + bool WriteBool(bool b) { + if (b) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); + } + else { + PutReserve(*os_, 5); + PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); + } + return true; + } + + bool WriteInt(int i) { + char buffer[11]; + const char* end = internal::i32toa(i, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint(unsigned u) { + char buffer[10]; + const char* end = internal::u32toa(u, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteInt64(int64_t i64) { + char buffer[21]; + const char* end = internal::i64toa(i64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint64(uint64_t u64) { + char buffer[20]; + char* end = internal::u64toa(u64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + if (!(writeFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + + char buffer[25]; + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteString(const Ch* str, SizeType length) { + static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const char escape[256] = { +#define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 + 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 + Z16, Z16, // 30~4F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF +#undef Z16 + }; + + if (TargetEncoding::supportUnicode) + PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + else + PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." + + PutUnsafe(*os_, '\"'); + GenericStringStream is(str); + while (ScanWriteUnescapedString(is, length)) { + const Ch c = is.Peek(); + if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { + // Unicode escaping + unsigned codepoint; + if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) + return false; + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { + PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); + } + else { + RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); + // Surrogate pair + unsigned s = codepoint - 0x010000; + unsigned lead = (s >> 10) + 0xD800; + unsigned trail = (s & 0x3FF) + 0xDC00; + PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(lead ) & 15]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(trail ) & 15]); + } + } + else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { + is.Take(); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + if (escape[static_cast(c)] == 'u') { + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); + PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); + } + } + else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; + } + PutUnsafe(*os_, '\"'); + return true; + } + + bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { + return RAPIDJSON_LIKELY(is.Tell() < length); + } + + bool WriteStartObject() { os_->Put('{'); return true; } + bool WriteEndObject() { os_->Put('}'); return true; } + bool WriteStartArray() { os_->Put('['); return true; } + bool WriteEndArray() { os_->Put(']'); return true; } + + bool WriteRawValue(const Ch* json, size_t length) { + PutReserve(*os_, length); + for (size_t i = 0; i < length; i++) { + RAPIDJSON_ASSERT(json[i] != '\0'); + PutUnsafe(*os_, json[i]); + } + return true; + } + + void Prefix(Type type) { + (void)type; + if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root + Level* level = level_stack_.template Top(); + if (level->valueCount > 0) { + if (level->inArray) + os_->Put(','); // add comma if it is not the first element in array + else // in object + os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. + hasRoot_ = true; + } + } + + OutputStream* os_; + internal::Stack level_stack_; + int maxDecimalPlaces_; + bool hasRoot_; + +private: + // Prohibit copy constructor & assignment operator. + Writer(const Writer&); + Writer& operator=(const Writer&); +}; + +// Full specialization for StringStream to prevent memory copying + +template<> +inline bool Writer::WriteInt(int i) { + char *buffer = os_->Push(11); + const char* end = internal::i32toa(i, buffer); + os_->Pop(static_cast(11 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteUint(unsigned u) { + char *buffer = os_->Push(10); + const char* end = internal::u32toa(u, buffer); + os_->Pop(static_cast(10 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteInt64(int64_t i64) { + char *buffer = os_->Push(21); + const char* end = internal::i64toa(i64, buffer); + os_->Pop(static_cast(21 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteUint64(uint64_t u) { + char *buffer = os_->Push(20); + const char* end = internal::u64toa(u, buffer); + os_->Pop(static_cast(20 - (end - buffer))); + return true; +} + +template<> +inline bool Writer::WriteDouble(double d) { + if (internal::Double(d).IsNanOrInf()) { + // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). + if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) { + PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; + } + if (internal::Double(d).Sign()) { + PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; + } + + char *buffer = os_->Push(25); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + os_->Pop(static_cast(25 - (end - buffer))); + return true; +} + +#if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (; p != endAligned; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + SizeType len; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + len = offset; +#else + len = static_cast(__builtin_ffs(r) - 1); +#endif + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#endif // defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + +RAPIDJSON_NAMESPACE_END + +#ifdef _MSC_VER +RAPIDJSON_DIAG_POP +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/ext/librethinkdbxx/src/term.cc b/ext/librethinkdbxx/src/term.cc new file mode 100644 index 00000000..711ef27d --- /dev/null +++ b/ext/librethinkdbxx/src/term.cc @@ -0,0 +1,285 @@ +#include +#include + +#include "term.h" +#include "json_p.h" + +namespace RethinkDB { + +using TT = Protocol::Term::TermType; + +struct { + Datum operator() (const Array& array) { + Array copy; + copy.reserve(array.size()); + for (const auto& it : array) { + copy.emplace_back(it.apply(*this)); + } + return Datum(Array{TT::MAKE_ARRAY, std::move(copy)}); + } + Datum operator() (const Object& object) { + Object copy; + for (const auto& it : object) { + copy.emplace(it.first, it.second.apply(*this)); + } + return std::move(copy); + } + template + Datum operator() (T&& atomic) { + return Datum(std::forward(atomic)); + } +} datum_to_term; + +Term::Term(Datum&& datum_) : datum(datum_.apply(datum_to_term)) { } +Term::Term(const Datum& datum_) : datum(datum_.apply(datum_to_term)) { } + +Term::Term(Term&& orig, OptArgs&& new_optargs) : datum(Nil()) { + Datum* cur = orig.datum.get_nth(2); + Object optargs; + free_vars = std::move(orig.free_vars); + if (cur) { + optargs = std::move(cur->extract_object()); + } + for (auto& it : new_optargs) { + optargs.emplace(std::move(it.first), alpha_rename(std::move(it.second))); + } + datum = Array{ std::move(orig.datum.extract_nth(0)), std::move(orig.datum.extract_nth(1)), std::move(optargs) }; +} + +Term nil() { + return Term(Nil()); +} + +Cursor Term::run(Connection& conn, OptArgs&& opts) { + if (!free_vars.empty()) { + throw Error("run: term has free variables"); + } + + return conn.start_query(this, std::move(opts)); +} + +struct { + Datum operator() (Object&& object, const std::map& subst, bool) { + Object ret; + for (auto& it : object) { + ret.emplace(std::move(it.first), std::move(it.second).apply(*this, subst, false)); + } + return ret; + } + Datum operator() (Array&& array, const std::map& subst, bool args) { + if (!args) { + double cmd = array[0].extract_number(); + if (cmd == static_cast(TT::VAR)) { + double var = array[1].extract_nth(0).extract_number(); + auto it = subst.find(static_cast(var)); + if (it != subst.end()) { + return Array{ TT::VAR, { it->second }}; + } + } + if (array.size() == 2) { + return Array{ std::move(array[0]), std::move(array[1]).apply(*this, subst, true) }; + } else { + return Array{ + std::move(array[0]), + std::move(array[1]).apply(*this, subst, true), + std::move(array[2]).apply(*this, subst, false) }; + } + } else { + Array ret; + for (auto& it : array) { + ret.emplace_back(std::move(it).apply(*this, subst, false)); + } + return ret; + } + } + template + Datum operator() (T&& a, const std::map&, bool) { + return std::move(a); + } +} alpha_renamer; + +static int new_var_id(const std::map& vars) { + while (true) { + int id = gen_var_id(); + if (vars.find(id) == vars.end()) { + return id; + } + } +} + +Datum Term::alpha_rename(Term&& term) { + if (free_vars.empty()) { + free_vars = std::move(term.free_vars); + return std::move(term.datum); + } + + std::map subst; + for (auto it = term.free_vars.begin(); it != term.free_vars.end(); ++it) { + auto var = free_vars.find(it->first); + if (var == free_vars.end()) { + free_vars.emplace(it->first, it->second); + } else if (var->second != it->second) { + int id = new_var_id(free_vars); + subst.emplace(it->first, id); + free_vars.emplace(id, it->second); + } + } + if (subst.empty()) { + return std::move(term.datum); + } else { + return term.datum.apply(alpha_renamer, subst, false); + } +} + +int gen_var_id() { + return ::random() % (1<<30); +} + +C0_IMPL(db_list, DB_LIST) +C0_IMPL(table_list, TABLE_LIST) +C0_IMPL(random, RANDOM) +C0_IMPL(now, NOW) +C0_IMPL(range, RANGE) +C0_IMPL(error, ERROR) +C0_IMPL(uuid, UUID) +C0_IMPL(literal, LITERAL) +CO0_IMPL(wait, WAIT) +C0_IMPL(rebalance, REBALANCE) +CO0_IMPL(random, RANDOM) + +Term row(TT::IMPLICIT_VAR, {}); +Term minval(TT::MINVAL, {}); +Term maxval(TT::MAXVAL, {}); + +Term binary(const std::string& data) { + return expr(Binary(data)); +} + +Term binary(std::string&& data) { + return expr(Binary(data)); +} + +Term binary(const char* data) { + return expr(Binary(data)); +} + +struct { + bool operator() (const Object& object) { + for (const auto& it : object) { + if (it.second.apply(*this)) { + return true; + } + } + return false; + } + bool operator() (const Array& array) { + int type = *array[0].get_number(); + if (type == static_cast(TT::IMPLICIT_VAR)) { + return true; + } + if (type == static_cast(TT::FUNC)) { + return false; + } + for (const auto& it : *array[1].get_array()) { + if (it.apply(*this)) { + return true; + } + } + if (array.size() == 3) { + return array[2].apply(*this); + } else { + return false; + } + } + template + bool operator() (T) { + return false; + } +} needs_func_wrap; + +Term Term::func_wrap(Term&& term) { + if (term.datum.apply(needs_func_wrap)) { + return Term(TT::FUNC, {expr(Array{new_var_id(term.free_vars)}), std::move(term)}); + } + return term; +} + +Term Term::func_wrap(const Term& term) { + if (term.datum.apply(needs_func_wrap)) { + // TODO return Term(TT::FUNC, {expr(Array{new_var_id(Term.free_vars)}), Term.copy()}); + return Term(Nil()); + } + return term; +} + +Term Term::make_object(std::vector&& args) { + if (args.size() % 2 != 0) { + return Term(TT::OBJECT, std::move(args)); + } + std::set keys; + for (auto it = args.begin(); it != args.end() && it + 1 != args.end(); it += 2) { + std::string* key = it->datum.get_string(); + if (!key || keys.count(*key)) { + return Term(TT::OBJECT, std::move(args)); + } + keys.insert(*key); + } + Term ret{Nil()}; + Object object; + for (auto it = args.begin(); it != args.end(); it += 2) { + std::string* key = it->datum.get_string(); + object.emplace(std::move(*key), ret.alpha_rename(std::move(*(it + 1)))); + } + ret.datum = std::move(object); + return ret; +} + +Term Term::make_binary(Term&& term) { + std::string* string = term.datum.get_string(); + if (string) { + return expr(Binary(std::move(*string))); + } + return Term(TT::BINARY, std::vector{term}); +} + +Term::Term(OptArgs&& optargs) : datum(Nil()) { + Object oargs; + for (auto& it : optargs) { + oargs.emplace(it.first, alpha_rename(std::move(it.second))); + } + datum = std::move(oargs); +} + +OptArgs optargs() { + return OptArgs{}; +} + +Term january(TT::JANUARY, {}); +Term february(TT::FEBRUARY, {}); +Term march(TT::MARCH, {}); +Term april(TT::APRIL, {}); +Term may(TT::MAY, {}); +Term june(TT::JUNE, {}); +Term july(TT::JULY, {}); +Term august(TT::AUGUST, {}); +Term september(TT::SEPTEMBER, {}); +Term october(TT::OCTOBER, {}); +Term november(TT::NOVEMBER, {}); +Term december(TT::DECEMBER, {}); +Term monday(TT::MONDAY, {}); +Term tuesday(TT::TUESDAY, {}); +Term wednesday(TT::WEDNESDAY, {}); +Term thursday(TT::THURSDAY, {}); +Term friday(TT::FRIDAY, {}); +Term saturday(TT::SATURDAY, {}); +Term sunday(TT::SUNDAY, {}); + +Term Term::copy() const { + return *this; +} + +Datum Term::get_datum() const { + return datum; +} + +} diff --git a/ext/librethinkdbxx/src/term.h b/ext/librethinkdbxx/src/term.h new file mode 100644 index 00000000..cdee5ec4 --- /dev/null +++ b/ext/librethinkdbxx/src/term.h @@ -0,0 +1,592 @@ +#pragma once + +#include "datum.h" +#include "connection.h" +#include "protocol_defs.h" +#include "cursor.h" + +namespace RethinkDB { + +using TT = Protocol::Term::TermType; + +class Term; +class Var; + +// An alias for the Term constructor +template +Term expr(T&&); + +int gen_var_id(); + +// Can be used as the last argument to some ReQL commands that expect named arguments +using OptArgs = std::map; + +// Represents a ReQL Term (RethinkDB Query Language) +// Designed to be used with r-value *this +class Term { +public: + Term(const Term& other) = default; + Term(Term&& other) = default; + Term& operator= (const Term& other) = default; + Term& operator= (Term&& other) = default; + + explicit Term(Datum&&); + explicit Term(const Datum&); + explicit Term(OptArgs&&); + + // Create a copy of the Term + Term copy() const; + + Term(std::function f) : datum(Nil()) { set_function>(f); } + Term(std::function f) : datum(Nil()) { set_function, 0>(f); } + Term(std::function f) : datum(Nil()) { set_function, 0, 1>(f); } + Term(std::function f) : datum(Nil()) { set_function, 0, 1, 2>(f); } + Term(Protocol::Term::TermType type, std::vector&& args) : datum(Array()) { + Array dargs; + for (auto& it : args) { + dargs.emplace_back(alpha_rename(std::move(it))); + } + datum = Datum(Array{ type, std::move(dargs) }); + } + + Term(Protocol::Term::TermType type, std::vector&& args, OptArgs&& optargs) : datum(Array()) { + Array dargs; + for (auto& it : args) { + dargs.emplace_back(alpha_rename(std::move(it))); + } + Object oargs; + for (auto& it : optargs) { + oargs.emplace(it.first, alpha_rename(std::move(it.second))); + } + datum = Array{ type, std::move(dargs), std::move(oargs) }; + } + + // Used internally to support row + static Term func_wrap(Term&&); + static Term func_wrap(const Term&); + + + // These macros are used to define most ReQL commands + // * Cn represents a method with n arguments + // * COn represents a method with n arguments and optional named arguments + // * C_ represents a method with any number of arguments + // Each method is implemented twice, once with r-value *this, and once with const *this + // The third argument, wrap, allows converting arguments into functions if they contain row + +#define C0(name, type) \ + Term name() && { return Term(TT::type, std::vector{ std::move(*this) }); } \ + Term name() const & { return Term(TT::type, std::vector{ *this }); } +#define C1(name, type, wrap) \ + template \ + Term name(T&& a) && { return Term(TT::type, std::vector{ std::move(*this), wrap(expr(std::forward(a))) }); } \ + template \ + Term name(T&& a) const & { return Term(TT::type, std::vector{ *this, wrap(expr(std::forward(a))) }); } +#define C2(name, type) \ + template Term name(T&& a, U&& b) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + expr(std::forward(a)), expr(std::forward(b)) }); } \ + template Term name(T&& a, U&& b) const & { \ + return Term(TT::type, std::vector{ *this, \ + expr(std::forward(a)), expr(std::forward(b)) }); } +#define C_(name, type, wrap) \ + template Term name(T&& ...a) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a)))... }); } \ + template Term name(T&& ...a) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a)))... }); } +#define CO0(name, type) \ + Term name(OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this) }, std::move(optarg)); } \ + Term name(OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this }, std::move(optarg)); } +#define CO1(name, type, wrap) \ + template Term name(T&& a, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))) }, std::move(optarg)); } \ + template Term name(T&& a, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))) }, std::move(optarg)); } +#define CO2(name, type, wrap) \ + template Term name(T&& a, U&& b, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))) }, std::move(optarg)); } \ + template Term name(T&& a, U&& b, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))) }, std::move(optarg)); } +#define CO3(name, type, wrap) \ + template Term name(T&& a, U&& b, V&& c, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c))) }, std::move(optarg)); } \ + template Term name(T&& a, U&& b, V&& c, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c)))}, std::move(optarg)); } +#define CO4(name, type, wrap) \ + template Term name(T&& a, U&& b, V&& c, W&& d, OptArgs&& optarg = {}) && { \ + return Term(TT::type, std::vector{ std::move(*this), \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c))), wrap(expr(std::forward(d))) }, std::move(optarg)); } \ + template Term name(T&& a, U&& b, V&& c, W&& d, OptArgs&& optarg = {}) const & { \ + return Term(TT::type, std::vector{ *this, \ + wrap(expr(std::forward(a))), wrap(expr(std::forward(b))), \ + wrap(expr(std::forward(c))), wrap(expr(std::forward(d))) }, std::move(optarg)); } +#define CO_(name, type, wrap) \ + C_(name, type, wrap) \ + CO0(name, type) \ + CO1(name, type, wrap) \ + CO2(name, type, wrap) \ + CO3(name, type, wrap) \ + CO4(name, type, wrap) +#define no_wrap(x) x + + CO1(table_create, TABLE_CREATE, no_wrap) + C1(table_drop, TABLE_DROP, no_wrap) + C0(table_list, TABLE_LIST) + CO1(index_create, INDEX_CREATE, no_wrap) + CO2(index_create, INDEX_CREATE, func_wrap) + C1(index_drop, INDEX_DROP, no_wrap) + C0(index_list, INDEX_LIST) + CO2(index_rename, INDEX_RENAME, no_wrap) + C_(index_status, INDEX_STATUS, no_wrap) + C_(index_wait, INDEX_WAIT, no_wrap) + CO0(changes, CHANGES) + CO1(insert, INSERT, no_wrap) + CO1(update, UPDATE, func_wrap) + CO1(replace, REPLACE, func_wrap) + CO0(delete_, DELETE) + C0(sync, SYNC) + CO1(table, TABLE, no_wrap) + C1(get, GET, no_wrap) + CO_(get_all, GET_ALL, no_wrap) + CO2(between, BETWEEN, no_wrap) + CO1(filter, FILTER, func_wrap) + C2(inner_join, INNER_JOIN) + C2(outer_join, OUTER_JOIN) + CO2(eq_join, EQ_JOIN, func_wrap) + C0(zip, ZIP) + C_(map, MAP, func_wrap) + C_(with_fields, WITH_FIELDS, no_wrap) + C1(concat_map, CONCAT_MAP, func_wrap) + CO_(order_by, ORDER_BY, func_wrap) + C1(skip, SKIP, no_wrap) + C1(limit, LIMIT, no_wrap) + CO1(slice, SLICE, no_wrap) + CO2(slice, SLICE, no_wrap) + C1(nth, NTH, no_wrap) + C1(offsets_of, OFFSETS_OF, func_wrap) + C0(is_empty, IS_EMPTY) + CO_(union_, UNION, no_wrap) + C1(sample, SAMPLE, no_wrap) + CO_(group, GROUP, func_wrap) + C0(ungroup, UNGROUP) + C1(reduce, REDUCE, no_wrap) + CO2(fold, FOLD, no_wrap) + C0(count, COUNT) + C1(count, COUNT, func_wrap) + C0(sum, SUM) + C1(sum, SUM, func_wrap) + C0(avg, AVG) + C1(avg, AVG, func_wrap) + C1(min, MIN, func_wrap) + CO0(min, MIN) + C1(max, MAX, func_wrap) + CO0(max, MAX) + CO0(distinct, DISTINCT) + C_(contains, CONTAINS, func_wrap) + C_(pluck, PLUCK, no_wrap) + C_(without, WITHOUT, no_wrap) + C_(merge, MERGE, func_wrap) + C1(append, APPEND, no_wrap) + C1(prepend, PREPEND, no_wrap) + C1(difference, DIFFERENCE, no_wrap) + C1(set_insert, SET_INSERT, no_wrap) + C1(set_union, SET_UNION, no_wrap) + C1(set_intersection, SET_INTERSECTION, no_wrap) + C1(set_difference, SET_DIFFERENCE, no_wrap) + C1(operator[], BRACKET, no_wrap) + C1(get_field, GET_FIELD, no_wrap) + C_(has_fields, HAS_FIELDS, no_wrap) + C2(insert_at, INSERT_AT) + C2(splice_at, SPLICE_AT) + C1(delete_at, DELETE_AT, no_wrap) + C2(delete_at, DELETE_AT) + C2(change_at, CHANGE_AT) + C0(keys, KEYS) + C1(match, MATCH, no_wrap) + C0(split, SPLIT) + C1(split, SPLIT, no_wrap) + C2(split, SPLIT) + C0(upcase, UPCASE) + C0(downcase, DOWNCASE) + C_(add, ADD, no_wrap) + C1(operator+, ADD, no_wrap) + C_(sub, SUB, no_wrap) + C1(operator-, SUB, no_wrap) + C_(mul, MUL, no_wrap) + C1(operator*, MUL, no_wrap) + C_(div, DIV, no_wrap) + C1(operator/, DIV, no_wrap) + C1(mod, MOD, no_wrap) + C1(operator%, MOD, no_wrap) + C_(and_, AND, no_wrap) + C1(operator&&, AND, no_wrap) + C_(or_, OR, no_wrap) + C1(operator||, OR, no_wrap) + C1(eq, EQ, no_wrap) + C1(operator==, EQ, no_wrap) + C1(ne, NE, no_wrap) + C1(operator!=, NE, no_wrap) + C1(gt, GT, no_wrap) + C1(operator>, GT, no_wrap) + C1(ge, GE, no_wrap) + C1(operator>=, GE, no_wrap) + C1(lt, LT, no_wrap) + C1(operator<, LT, no_wrap) + C1(le, LE, no_wrap) + C1(operator<=, LE, no_wrap) + C0(not_, NOT) + C0(operator!, NOT) + C1(in_timezone, IN_TIMEZONE, no_wrap) + C0(timezone, TIMEZONE) + CO2(during, DURING, no_wrap) + C0(date, DATE) + C0(time_of_day, TIME_OF_DAY) + C0(year, YEAR) + C0(month, MONTH) + C0(day, DAY) + C0(day_of_week, DAY_OF_WEEK) + C0(day_of_year, DAY_OF_YEAR) + C0(hours, HOURS) + C0(minutes, MINUTES) + C0(seconds, SECONDS) + C0(to_iso8601, TO_ISO8601) + C0(to_epoch_time, TO_EPOCH_TIME) + C1(for_each, FOR_EACH, func_wrap) + C1(default_, DEFAULT, no_wrap) + CO1(js, JAVASCRIPT, no_wrap) + C1(coerce_to, COERCE_TO, no_wrap) + C0(type_of, TYPE_OF) + C0(info, INFO) + C0(to_json, TO_JSON_STRING) + C0(to_json_string, TO_JSON_STRING) + C1(distance, DISTANCE, no_wrap) + C0(fill, FILL) + C0(to_geojson, TO_GEOJSON) + CO1(get_intersecting, GET_INTERSECTING, no_wrap) + CO1(get_nearest, GET_NEAREST, no_wrap) + C1(includes, INCLUDES, no_wrap) + C1(intersects, INTERSECTS, no_wrap) + C1(polygon_sub, POLYGON_SUB, no_wrap) + C0(config, CONFIG) + C0(rebalance, REBALANCE) + CO0(reconfigure, RECONFIGURE) + C0(status, STATUS) + CO0(wait, WAIT) + C0(floor, FLOOR) + C0(ceil, CEIL) + C0(round, ROUND) + C0(values, VALUES) + + // The expansion of this macro fails to compile on some versions of GCC and Clang: + // C_(operator(), FUNCALL, no_wrap) + // The std::enable_if makes the error go away + + // $doc(do) + + template + typename std::enable_if::value, Term>::type + operator() (T&& a, U&& ...b) && { + return Term(TT::FUNCALL, std::vector{ + std::move(*this), + expr(std::forward(a)), + expr(std::forward(b))... }); + } + template + typename std::enable_if::value, Term>::type + operator() (T&& a, U&& ...b) const & { + return Term(TT::FUNCALL, std::vector{ + *this, + expr(std::forward(a)), + expr(std::forward(b))... }); + } + +#undef C0 +#undef C1 +#undef C2 +#undef C_ +#undef CO0 +#undef CO1 +#undef CO2 + + // Send the term to the server and return the results. + // Errors returned by the server are thrown. + Cursor run(Connection&, OptArgs&& args = {}); + + // $doc(do) + template + Term do_(T&& ...a) && { + auto list = { std::move(*this), Term::func_wrap(expr(std::forward(a)))... }; + std::vector args; + args.reserve(list.size() + 1); + args.emplace_back(func_wrap(std::move(*(list.end()-1)))); + for (auto it = list.begin(); it + 1 != list.end(); ++it) { + args.emplace_back(std::move(*it)); + } + return Term(TT::FUNCALL, std::move(args)); + } + + // Adds optargs to an already built term + Term opt(OptArgs&& optargs) && { + return Term(std::move(*this), std::move(optargs)); + } + + // Used internally to implement object() + static Term make_object(std::vector&&); + + // Used internally to implement array() + static Term make_binary(Term&&); + + Datum get_datum() const; + +private: + friend class Var; + friend class Connection; + friend struct Query; + + template + Var mkvar(std::vector& vars); + + template + void set_function(F); + + Datum alpha_rename(Term&&); + + Term(Term&& orig, OptArgs&& optargs); + + std::map free_vars; + Datum datum; +}; + +// A term representing null +Term nil(); + +template +Term expr(T&& a) { + return Term(std::forward(a)); +} + +// Represents a ReQL variable. +// This type is passed to functions used in ReQL queries. +class Var { +public: + // Convert to a term + Term operator*() const { + Term term(TT::VAR, std::vector{expr(*id)}); + term.free_vars = {{*id, id}}; + return term; + } + + Var(int* id_) : id(id_) { } +private: + int* id; +}; + +template +Var Term::mkvar(std::vector& vars) { + int id = gen_var_id(); + vars.push_back(id); + return Var(&*vars.rbegin()); +} + +template +void Term::set_function(F f) { + std::vector vars; + vars.reserve(sizeof...(N)); + std::vector args = { mkvar(vars)... }; + Term body = f(args[N] ...); + + int* low = &*vars.begin(); + int* high = &*(vars.end() - 1); + for (auto it = body.free_vars.begin(); it != body.free_vars.end(); ) { + if (it->second >= low && it->second <= high) { + if (it->first != *it->second) { + throw Error("Internal error: variable index mis-match"); + } + ++it; + } else { + free_vars.emplace(*it); + ++it; + } + } + datum = Array{TT::FUNC, Array{Array{TT::MAKE_ARRAY, vars}, body.datum}}; +} + +// These macros are similar to those defined above, but for top-level ReQL operations + +#define C0(name) Term name(); +#define C0_IMPL(name, type) Term name() { return Term(TT::type, std::vector{}); } +#define CO0(name) Term name(OptArgs&& optargs = {}); +#define CO0_IMPL(name, type) Term name(OptArgs&& optargs) { return Term(TT::type, std::vector{}, std::move(optargs)); } +#define C1(name, type, wrap) template Term name(T&& a) { \ + return Term(TT::type, std::vector{ wrap(expr(std::forward(a))) }); } +#define C2(name, type) template Term name(T&& a, U&& b) { \ + return Term(TT::type, std::vector{ expr(std::forward(a)), expr(std::forward(b)) }); } +#define C3(name, type) template \ + Term name(A&& a, B&& b, C&& c) { return Term(TT::type, std::vector{ \ + expr(std::forward(a)), expr(std::forward(b)), expr(std::forward(c)) }); } +#define C4(name, type) template \ + Term name(A&& a, B&& b, C&& c, D&& d) { return Term(TT::type, std::vector{ \ + expr(std::forward(a)), expr(std::forward(b)), \ + expr(std::forward(c)), expr(std::forward(d))}); } +#define C7(name, type) template \ + Term name(A&& a, B&& b, C&& c, D&& d, E&& e, F&& f, G&& g) { return Term(TT::type, std::vector{ \ + expr(std::forward(a)), expr(std::forward(b)), expr(std::forward(c)), \ + expr(std::forward(d)), expr(std::forward(e)), expr(std::forward(f)), \ + expr(std::forward(g))}); } +#define C_(name, type, wrap) template Term name(T&& ...a) { \ + return Term(TT::type, std::vector{ wrap(expr(std::forward(a)))... }); } +#define CO1(name, type, wrap) template Term name(T&& a, OptArgs&& optarg = {}) { \ + return Term(TT::type, std::vector{ wrap(expr(std::forward(a)))}, std::move(optarg)); } +#define CO2(name, type) template Term name(T&& a, U&& b, OptArgs&& optarg = {}) { \ + return Term(TT::type, std::vector{ expr(std::forward(a)), expr(std::forward(b))}, std::move(optarg)); } +#define func_wrap Term::func_wrap + +C1(db_create, DB_CREATE, no_wrap) +C1(db_drop, DB_DROP, no_wrap) +C0(db_list) +CO1(table_create, TABLE_CREATE, no_wrap) +C1(table_drop, TABLE_DROP, no_wrap) +C0(table_list) +C1(db, DB, no_wrap) +CO1(table, TABLE, no_wrap) +C_(add, ADD, no_wrap) +C2(sub, SUB) +C_(mul, MUL, no_wrap) +C_(div, DIV, no_wrap) +C2(mod, MOD) +C_(and_, AND, no_wrap) +C_(or_, OR, no_wrap) +C2(eq, EQ) +C2(ne, NE) +C2(gt, GT) +C2(ge, GE) +C2(lt, LT) +C2(le, LE) +C1(not_, NOT, no_wrap) +CO0(random) +CO1(random, RANDOM, no_wrap) +CO2(random, RANDOM) +C0(now) +C4(time, TIME) +C7(time, TIME) +C1(epoch_time, EPOCH_TIME, no_wrap) +CO1(iso8601, ISO8601, no_wrap) +CO1(js, JAVASCRIPT, no_wrap) +C1(args, ARGS, no_wrap) +C_(branch, BRANCH, no_wrap) +C0(range) +C1(range, RANGE, no_wrap) +C2(range, RANGE) +C0(error) +C1(error, ERROR, no_wrap) +C1(json, JSON, no_wrap) +CO1(http, HTTP, func_wrap) +C0(uuid) +C1(uuid, UUID, no_wrap) +CO2(circle, CIRCLE) +C1(geojson, GEOJSON, no_wrap) +C_(line, LINE, no_wrap) +C2(point, POINT) +C_(polygon, POLYGON, no_wrap) +C_(array, MAKE_ARRAY, no_wrap) +C1(desc, DESC, func_wrap) +C1(asc, ASC, func_wrap) +C0(literal) +C1(literal, LITERAL, no_wrap) +C1(type_of, TYPE_OF, no_wrap) +C_(map, MAP, func_wrap) +C1(floor, FLOOR, no_wrap) +C1(ceil, CEIL, no_wrap) +C1(round, ROUND, no_wrap) +C_(union_, UNION, no_wrap) +C_(group, GROUP, func_wrap) +C1(count, COUNT, no_wrap) +C_(count, COUNT, func_wrap) +C1(sum, SUM, no_wrap) +C_(sum, SUM, func_wrap) +C1(avg, AVG, no_wrap) +C_(avg, AVG, func_wrap) +C1(min, MIN, no_wrap) +C_(min, MIN, func_wrap) +C1(max, MAX, no_wrap) +C_(max, MAX, func_wrap) +C1(distinct, DISTINCT, no_wrap) +C1(contains, CONTAINS, no_wrap) +C_(contains, CONTAINS, func_wrap) + +#undef C0 +#undef C1 +#undef C2 +#undef C3 +#undef C4 +#undef C7 +#undef C_ +#undef CO1 +#undef CO2 +#undef func_wrap + +// $doc(do) +template +Term do_(R&& a, T&& ...b) { + return expr(std::forward(a)).do_(std::forward(b)...); +} + +// $doc(object) +template +Term object(T&& ...a) { + return Term::make_object(std::vector{ expr(std::forward(a))... }); +} + +// $doc(binary) +template +Term binary(T&& a) { + return Term::make_binary(expr(std::forward(a))); +} + +// Construct an empty optarg +OptArgs optargs(); + +// Construct an optarg made out of pairs of arguments +// For example: optargs("k1", v1, "k2", v2) +template +OptArgs optargs(const char* key, V&& val, T&& ...rest) { + OptArgs opts = optargs(rest...); + opts.emplace(key, expr(std::forward(val))); + return opts; +} + +extern Term row; +extern Term maxval; +extern Term minval; +extern Term january; +extern Term february; +extern Term march; +extern Term april; +extern Term may; +extern Term june; +extern Term july; +extern Term august; +extern Term september; +extern Term october; +extern Term november; +extern Term december; +extern Term monday; +extern Term tuesday; +extern Term wednesday; +extern Term thursday; +extern Term friday; +extern Term saturday; +extern Term sunday; +} diff --git a/ext/librethinkdbxx/src/types.cc b/ext/librethinkdbxx/src/types.cc new file mode 100644 index 00000000..ea9becaf --- /dev/null +++ b/ext/librethinkdbxx/src/types.cc @@ -0,0 +1,47 @@ +#include + +#include "types.h" +#include "error.h" + +namespace RethinkDB { + +bool Time::parse_utc_offset(const std::string& string, double* offset) { + const char *s = string.c_str(); + double sign = 1; + switch (s[0]) { + case '-': + sign = -1; + case '+': + ++s; + break; + case 0: + return false; + } + for (int i = 0; i < 5; ++i) { + if (s[i] == 0) return false; + if (i == 2) continue; + if (s[i] < '0' || s[i] > '9') return false; + } + if (s[2] != ':') return false; + *offset = sign * ((s[0] - '0') * 36000 + (s[1] - '0') * 3600 + (s[3] - '0') * 600 + (s[4] - '0') * 60); + return true; +} + +double Time::parse_utc_offset(const std::string& string) { + double out; + if (!parse_utc_offset(string, &out)) { + throw Error("invalid utc offset `%s'", string.c_str()); + } + return out; +} + +std::string Time::utc_offset_string(double offset) { + char buf[8]; + int hour = offset / 3600; + int minutes = std::abs(static_cast(offset / 60)) % 60; + int n = snprintf(buf, 7, "%+03d:%02d", hour, minutes); + buf[n] = 0; + return std::string(buf); +} + +} diff --git a/ext/librethinkdbxx/src/types.h b/ext/librethinkdbxx/src/types.h new file mode 100644 index 00000000..ac35a871 --- /dev/null +++ b/ext/librethinkdbxx/src/types.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + +namespace RethinkDB { + +class Datum; + +// Represents a null datum +struct Nil { }; + +using Array = std::vector; +using Object = std::map; + +// Represents a string of bytes. Plain std::strings are passed on to the server as utf-8 strings +struct Binary { + bool operator== (const Binary& other) const { + return data == other.data; + } + + Binary(const std::string& data_) : data(data_) { } + Binary(std::string&& data_) : data(std::move(data_)) { } + std::string data; +}; + +// Represents a point in time as +// * A floating amount of seconds since the UNIX epoch +// * And a timezone offset represented as seconds relative to UTC +struct Time { + Time(double epoch_time_, double utc_offset_ = 0) : + epoch_time(epoch_time_), utc_offset(utc_offset_) { } + + static Time now() { + return Time(time(NULL)); + } + + static bool parse_utc_offset(const std::string&, double*); + static double parse_utc_offset(const std::string&); + static std::string utc_offset_string(double); + + double epoch_time; + double utc_offset; +}; + +// Not implemented +class Point; +class Line; +class Polygon; + +} diff --git a/ext/librethinkdbxx/src/utils.cc b/ext/librethinkdbxx/src/utils.cc new file mode 100644 index 00000000..5a2c244d --- /dev/null +++ b/ext/librethinkdbxx/src/utils.cc @@ -0,0 +1,153 @@ +#include "utils.h" +#include "error.h" + +namespace RethinkDB { + +size_t utf8_encode(unsigned int code, char* buf) { + if (!(code & ~0x7F)) { + buf[0] = code; + return 1; + } else if (!(code & ~0x7FF)) { + buf[0] = 0xC0 | (code >> 6); + buf[1] = 0x80 | (code & 0x3F); + return 2; + } else if (!(code & ~0xFFFF)) { + buf[0] = 0xE0 | (code >> 12); + buf[1] = 0x80 | ((code >> 6) & 0x3F); + buf[2] = 0x80 | (code & 0x3F); + return 3; + } else if (!(code & ~0x1FFFFF)) { + buf[0] = 0xF0 | (code >> 18); + buf[1] = 0x80 | ((code >> 12) & 0x3F); + buf[2] = 0x80 | ((code >> 6) & 0x3F); + buf[3] = 0x80 | (code & 0x3F); + return 4; + } else if (!(code & ~0x3FFFFFF)) { + buf[0] = 0xF8 | (code >> 24); + buf[1] = 0x80 | ((code >> 18) & 0x3F); + buf[2] = 0x80 | ((code >> 12) & 0x3F); + buf[3] = 0x80 | ((code >> 6) & 0x3F); + buf[4] = 0x80 | (code & 0x3F); + return 5; + } else if (!(code & ~0x7FFFFFFF)) { + buf[0] = 0xFC | (code >> 30); + buf[1] = 0x80 | ((code >> 24) & 0x3F); + buf[2] = 0x80 | ((code >> 18) & 0x3F); + buf[3] = 0x80 | ((code >> 12) & 0x3F); + buf[4] = 0x80 | ((code >> 6) & 0x3F); + buf[5] = 0x80 | (code & 0x3F); + return 6; + } else { + throw Error("Invalid unicode codepoint %ud", code); + } +} + +bool base64_decode(char c, int* out) { + if (c >= 'A' && c <= 'Z') { + *out = c - 'A'; + } else if (c >= 'a' && c <= 'z') { + *out = c - ('a' - 26); + } else if (c >= '0' && c <= '9') { + *out = c - ('0' - 52); + } else if (c == '+') { + *out = 62; + } else if (c == '/') { + *out = 63; + } else { + return false; + } + return true; +} + +bool base64_decode(const std::string& in, std::string& out) { + out.clear(); + out.reserve(in.size() * 3 / 4); + auto read = in.begin(); + while (true) { + int c[4]; + int end = 4; + for (int i = 0; i < 4; i++) { + while (true) { + if (read == in.end()) { + c[i] = 0; + end = i; + i = 3; + break; + } else if (base64_decode(*read, &c[i])) { + ++read; + break; + } else { + ++read; + } + } + } + if (end == 1) return false; + int val = c[0] << 18 | c[1] << 12 | c[2] << 6 | c[3]; + if (end > 1) out.append(1, val >> 16); + if (end > 2) out.append(1, val >> 8 & 0xFF); + if (end > 3) out.append(1, val & 0xFF); + if (end != 4) break; + } + return true; +} + +char base64_encode(unsigned int c) { + if (c < 26) { + return 'A' + c; + } else if (c < 52) { + return 'a' + c - 26; + } else if (c < 62) { + return '0' + c - 52; + } else if (c == 62) { + return '+'; + } else if (c == 63) { + return '/'; + } else { + throw Error("unreachable: base64 encoding %d", c); + } +} + +void base64_encode(unsigned int* c, int n, std::string& out) { + if (n == 0) { + return; + } + out.append(1, base64_encode(c[0] >> 2)); + out.append(1, base64_encode((c[0] & 0x3) << 4 | c[1] >> 4)); + if (n == 1) { + out.append("=="); + return; + } + out.append(1, base64_encode((c[1] & 0xF) << 2 | c[2] >> 6)); + if (n == 2) { + out.append("="); + return; + } + out.append(1, base64_encode(c[2] & 0x3F)); +} + +std::string base64_encode(const std::string& in) { + std::string out; + out.reserve(in.size() * 4 / 3 + in.size() / 48 + 3); + auto read = in.begin(); + while (true) { + for (int group = 0; group < 16; ++group) { + unsigned int c[3]; + int i = 0; + for (; i < 3; ++i) { + if (read == in.end()) { + c[i] = 0; + break; + } else { + c[i] = static_cast(*read++); + } + } + base64_encode(c, i, out); + if (i != 3) { + return out; + } + } + out.append("\n"); + } +} + +} diff --git a/ext/librethinkdbxx/src/utils.h b/ext/librethinkdbxx/src/utils.h new file mode 100644 index 00000000..04496e2c --- /dev/null +++ b/ext/librethinkdbxx/src/utils.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace RethinkDB { + +// The size of the longest UTF-8 encoded unicode codepoint +const size_t max_utf8_encoded_size = 6; + +// Decode a base64 string. Returns false on failure. +bool base64_decode(const std::string& in, std::string& out); +std::string base64_encode(const std::string&); + +// Encodes a single unicode codepoint into UTF-8. Returns the number of bytes written. +// Does not add a trailing null byte +size_t utf8_encode(unsigned int, char*); + +} diff --git a/ext/librethinkdbxx/test/bench.cc b/ext/librethinkdbxx/test/bench.cc new file mode 100644 index 00000000..e2843d4c --- /dev/null +++ b/ext/librethinkdbxx/test/bench.cc @@ -0,0 +1,58 @@ +/* +#include +#include +#include +#include + +namespace R = RethinkDB; +std::unique_ptr conn; + +int main() { + signal(SIGPIPE, SIG_IGN); + try { + conn = R::connect(); + } catch(const R::Error& error) { + printf("FAILURE: could not connect to localhost:28015: %s\n", error.message.c_str()); + return 1; + } + + try { + printf("running test...\n"); + auto start = std::chrono::steady_clock::now(); + R::Datum d = R::range(1, 1000000) + .map([]() { return R::object("test", "hello", "data", "world"); }) + .run(*conn); + auto end = std::chrono::steady_clock::now(); + auto diff = end - start; + + printf("result size: %d\n", (int)d.get_array()->size()); + printf("completed in %f ms\n", std::chrono::duration(diff).count()); + } catch (const R::Error& error) { + printf("FAILURE: uncaught exception: %s\n", error.message.c_str()); + return 1; + } +} +*/ + +#include +#include + +namespace R = RethinkDB; + +int main() { + auto conn = R::connect(); + if (!conn) { + std::cerr << "Could not connect to server\n"; + return 1; + } + + std::cout << "Connected" << std::endl; + R::Cursor databases = R::db_list().run(*conn); + for (R::Datum const& db : databases) { + std::cout << *db.get_string() << '\n'; + } + + return 0; +} + + diff --git a/ext/librethinkdbxx/test/gen_index_cxx.py b/ext/librethinkdbxx/test/gen_index_cxx.py new file mode 100644 index 00000000..220bb5a2 --- /dev/null +++ b/ext/librethinkdbxx/test/gen_index_cxx.py @@ -0,0 +1,11 @@ +from sys import argv +from re import sub + +print("#include \"testlib.h\""); +print("void run_upstream_tests() {") +for path in argv[1:]: + name = sub('/', '_', path.split('.')[0]) + print(" extern void %s();" % name) + print(" clean_slate();") + print(" %s();" % name) +print("}") diff --git a/ext/librethinkdbxx/test/test.cc b/ext/librethinkdbxx/test/test.cc new file mode 100644 index 00000000..fb5ca2fc --- /dev/null +++ b/ext/librethinkdbxx/test/test.cc @@ -0,0 +1,114 @@ +#include + +#include + +#include "testlib.h" + +extern void run_upstream_tests(); + +void test_json(const char* string, const char* ret = "") { + TEST_EQ(R::Datum::from_json(string).as_json().c_str(), ret[0] ? ret : string); +} + +void test_json_parse_print() { + enter_section("json"); + test_json("-0.0", "-0.0"); + test_json("null"); + test_json("1.2"); + test_json("1.2e20", "1.2e+20"); + test_json("true"); + test_json("false"); + test_json("\"\""); + test_json("\"\\u1234\"", "\"\u1234\""); + test_json("\"\\\"\""); + test_json("\"foobar\""); + test_json("[]"); + test_json("[1]"); + test_json("[1,2,3,4]"); + test_json("{}"); + test_json("{\"a\":1}"); + test_json("{\"a\":1,\"b\":2,\"c\":3}"); + exit_section(); +} + +void test_reql() { + enter_section("reql"); + TEST_EQ((R::expr(1) + 2).run(*conn), R::Datum(3)); + TEST_EQ(R::range(4).count().run(*conn), R::Datum(4)); + TEST_EQ(R::js("Math.abs")(-1).run(*conn), 1); + exit_section(); +} + +void test_cursor() { + enter_section("cursor"); + R::Cursor cursor = R::range(10000).run(*conn); + TEST_EQ(cursor.next(), 0); + R::Array array = cursor.to_array(); + TEST_EQ(array.size(), 9999); + TEST_EQ(*array.begin(), 1); + TEST_EQ(*array.rbegin(), 9999); + int i = 0; + R::range(3).run(*conn).each([&i](R::Datum&& datum){ + TEST_EQ(datum, i++); }); + exit_section(); +} + +void test_encode(const char* str, const char* b) { + TEST_EQ(R::base64_encode(str), b); +} + +void test_decode(const char* b, const char* str) { + std::string out; + TEST_EQ(R::base64_decode(b, out), true); + TEST_EQ(out, str); +} + +#define TEST_B64(a, b) test_encode(a, b); test_decode(b, a) + +void test_binary() { + enter_section("base64"); + TEST_B64("", ""); + TEST_B64("foo", "Zm9v"); + exit_section(); +} + +void test_issue28() { + enter_section("issue #28"); + std::vector expected{ "rethinkdb", "test" }; + std::vector dbs; + R::Cursor databases = R::db_list().run(*conn); + for (R::Datum const& db : databases) { + dbs.push_back(*db.get_string()); + } + + TEST_EQ(dbs, expected); + exit_section(); +} + +int main() { + signal(SIGPIPE, SIG_IGN); + srand(time(NULL)); + try { + conn = R::connect(); + } catch(const R::Error& error) { + printf("FAILURE: could not connect to localhost:28015: %s\n", error.message.c_str()); + return 1; + } + try { + //test_binary(); + //test_json_parse_print(); + //test_reql(); + //test_cursor(); + test_issue28(); + run_upstream_tests(); + } catch (const R::Error& error) { + printf("FAILURE: uncaught expception: %s\n", error.message.c_str()); + return 1; + } + if (!failed) { + printf("SUCCESS: %d tests passed\n", count); + } else { + printf("DONE: %d of %d tests failed\n", failed, count); + return 1; + } +} diff --git a/ext/librethinkdbxx/test/testlib.cc b/ext/librethinkdbxx/test/testlib.cc new file mode 100644 index 00000000..ddc9cc27 --- /dev/null +++ b/ext/librethinkdbxx/test/testlib.cc @@ -0,0 +1,356 @@ +#include +#include +#include +#include +#include + +#include "testlib.h" + +int verbosity = 0; + +int failed = 0; +int count = 0; +std::vector> section; + +std::unique_ptr conn; + +// std::string to_string(const R::Cursor&) { +// return ""; +// } + +std::string to_string(const R::Term& query) { + return to_string(query.get_datum()); +} + +std::string to_string(const R::Datum& datum) { + return datum.as_json(); +} + +std::string to_string(const R::Object& object) { + auto it = object.find("special"); + if (it != object.end()) { + std::string type = *(it->second).get_string(); + auto bag = object.find(type); + if (bag != object.end()) { + return to_string((R::Datum)bag->second); + } + } + + return to_string((R::Datum)object); +} + +std::string to_string(const R::Error& error) { + return "Error(\"" + error.message + "\")"; +} + +void enter_section(const char* name) { + if (verbosity == 0) { + section.emplace_back(name, true); + } else { + printf("%sSection %s\n", indent(), name); + section.emplace_back(name, false); + } +} + +void section_cleanup() { + R::db("test").table_list().for_each([=](R::Var table) { + return R::db("test").table_drop(*table); + }).run(*conn); +} + +void exit_section() { + section.pop_back(); +} + +std::string to_string(const err& error) { + return "Error(\"" + error.convert_type() + ": " + error.message + "\")"; +} + +bool equal(const R::Error& a, const err& b) { + // @TODO: I think the proper solution to this proble is to in fact create + // a hierarchy of exception types. This would not only simplify these + // error cases, but could be of great use to the user. + std::string error_type = b.convert_type(); + if (error_type == "ReqlServerCompileError" && + a.message.find("ReqlCompileError") != std::string::npos) { + return true; + } + + return b.trim_message(a.message) == (error_type + ": " + b.message); +} + +bool match(const char* pattern, const char* string) { + return std::regex_match(string, std::regex(pattern)); +} + +bool equal(const R::Error& a, const err_regex& b) { + if (b.message == "Object keys must be strings" && + a.message == "runtime error: Expected type STRING but found NUMBER.") { + return true; + } + return match(b.regex().c_str(), a.message.c_str()); +} + +std::string to_string(const err_regex& error) { + return "err_regex(" + error.type + ", " + error.message + ")"; +} + +R::Object partial(R::Object&& object) { + return R::Object{{"special", "partial"}, {"partial", std::move(object)}}; +} + +R::Datum uuid() { + return R::Object{{"special", "uuid"}}; +} + +R::Object arrlen(int n, R::Datum&& datum) { + return R::Object{{"special", "arrlen"},{"len",n},{"of",datum}}; +} + +R::Object arrlen(int n) { + return R::Object{{"special", "arrlen"},{"len",n}}; +} + +std::string repeat(std::string&& s, int n) { + std::string string; + string.reserve(n * s.size()); + for (int i = 0; i < n; ++i) { + string.append(s); + } + return string; +} + +R::Term fetch(R::Cursor& cursor, int count, double timeout) { + // printf("fetch(..., %d, %lf)\n", count, timeout); + R::Array array; + int deadline = time(NULL) + int(timeout); + for (int i = 0; count == -1 || i < count; ++i) { + // printf("fetching next (%d)\n", i); + time_t now = time(NULL); + if (now > deadline) break; + + try { + array.emplace_back(cursor.next(deadline - now)); + // printf("got %s\n", write_datum(array[array.size()-1]).c_str()); + } catch (const R::Error &e) { + if (e.message != "next: No more data") { + throw e; // rethrow + } + + break; + } catch (const R::TimeoutException &e){ + // printf("fetch timeout\n"); + break; + } + } + + return expr(std::move(array)); +} + +R::Object bag(R::Array&& array) { + return R::Object{{"special", "bag"}, {"bag", std::move(array)}}; +}; + +R::Object bag(R::Datum&& d) { + return R::Object{{"special", "bag"}, {"bag", std::move(d)}}; +}; + +std::string string_key(const R::Datum& datum) { + const std::string* string = datum.get_string(); + if (string) return *string; + return datum.as_json(); +} + +bool falsey(R::Datum&& datum) { + bool* boolean = datum.get_boolean(); + if (boolean) return !*boolean; + double* number = datum.get_number(); + if (number) return *number == 0; + return false; +} + +bool equal(const R::Datum& got, const R::Datum& expected) { + const std::string* string = expected.get_string(); + if (string) { + const R::Binary* binary = got.get_binary(); + if (binary) { + return *binary == R::Binary(*string); + } + } + if (expected.get_object() && expected.get_field("$reql_type$")) { + if (!got.get_field("$reql_type$")) { + R::Datum datum = got.to_raw(); + if (datum.get_field("$reql_type$")) { + return equal(datum, expected); + } + } + } + if (got.get_object() && got.get_field("$reql_type$")) { + const std::string* type = got.get_field("$reql_type$")->get_string(); + if (type && *type == "GROUPED_DATA" && + (!expected.get_object() || !expected.get_field("$reql_type$"))) { + const R::Array* data = got.get_field("data")->get_array(); + R::Object object; + for (R::Datum it : *data) { + object.emplace(string_key(it.extract_nth(0)), it.extract_nth(1)); + } + return equal(object, expected); + } + } + do { + if (!expected.get_object()) break; + if(!expected.get_field("special")) break; + const std::string* type = expected.get_field("special")->get_string(); + if (!type) break; + if (*type == "bag") { + const R::Datum* bag_datum = expected.get_field("bag"); + if (!bag_datum || !bag_datum->get_array()) { + break; + } + R::Array bag = *bag_datum->get_array(); + const R::Array* array = got.get_array(); + if (!array) { + return false; + } + if (bag.size() != array->size()) { + return false; + } + for (const auto& it : *array) { + auto ref = std::find(bag.begin(), bag.end(), it); + if (ref == bag.end()) return false; + bag.erase(ref); + } + return true; + } else if (*type == "arrlen") { + const R::Datum* len_datum = expected.get_field("len"); + if (!len_datum) break; + const double *len = len_datum->get_number(); + if (!len) break; + const R::Array* array = got.get_array(); + if (!array) break; + return array->size() == *len; + } else if (*type == "partial") { + const R::Object* object = got.get_object(); + if (object) { + const R::Datum* partial_datum = expected.get_field("partial"); + if (!partial_datum) break; + const R::Object* partial = partial_datum->get_object(); + if (!partial) break; + for (const auto& it : *partial) { + if (!object->count(it.first) || !equal((*object).at(it.first), it.second)) { + return false; + } + return true; + } + } + const R::Array* array = got.get_array(); + if (array) { + const R::Datum* partial_datum = expected.get_field("partial"); + if (!partial_datum) break; + const R::Array* partial = partial_datum->get_array(); + if (!partial) break; + + for (const auto& want : *partial) { + bool match = false; + for (const auto& have : *array) { + if (equal(have, want)) { + match = true; + break; + } + } + if (match == false) return false; + } + return true; + } + } else if(*type == "uuid") { + const std::string* string = got.get_string(); + if (string && string->size() == 36) { + return true; + } + } else if (*type == "regex") { + const R::Datum* regex_datum = expected.get_field("regex"); + if (!regex_datum) break; + const std::string* regex = regex_datum->get_string(); + if (!regex) break; + const std::string* str = got.get_string(); + if (!str) break; + return match(regex->c_str(), str->c_str()); + } + } while(0); + const R::Object* got_object = got.get_object(); + const R::Object* expected_object = expected.get_object(); + if (got_object && expected_object) { + R::Object have = *got_object; + for (const auto& it : *expected_object) { + auto other = have.find(it.first); + if (other == have.end()) return false; + if (!equal(other->second, it.second)) return false; + have.erase(other); + } + for (auto& it : have) { + if (!falsey(std::move(it.second))) { + return false; + } + } + return true; + } + const R::Array* got_array = got.get_array(); + const R::Array* expected_array = expected.get_array(); + if (got_array && expected_array) { + if (got_array->size() != expected_array->size()) return false; + for (R::Array::const_iterator i = got_array->begin(), j = expected_array->begin(); + i < got_array->end(); + i++, j++) { + if(!equal(*i, *j)) return false; + } + return true; + } + return got == expected; +} + +R::Object partial(R::Array&& array) { + return R::Object{{"special", "partial"}, {"partial", std::move(array)}}; +} + +R::Object regex(const char* pattern) { + return R::Object{{"special", "regex"}, {"regex", pattern}}; +} + +void clean_slate() { + R::table_list().for_each([](R::Var t){ return R::table_drop(*t); }); + R::db("rethinkdb").table("_debug_scratch").delete_().run(*conn); +} + +const char* indent() { + static const char spaces[] = " "; + return spaces + sizeof(spaces) - 1 - 2 * section.size(); +} + +std::string truncate(std::string&& string) { + if (string.size() > 200) { + return string.substr(0, 197) + "..."; + } + return string; +} + +int len(const R::Datum& d) { + const R::Array* arr = d.get_array(); + if (!arr) throw ("testlib: len: expected an array but got " + to_string(d)); + return arr->size(); +} + +R::Term wait(int n) { + std::this_thread::sleep_for(std::chrono::seconds(n)); + return R::expr(n); +} + +R::Datum nil = R::Nil(); + +R::Array append(R::Array lhs, R::Array rhs) { + if (lhs.empty()) { + return rhs; + } + lhs.reserve(lhs.size() + rhs.size()); + std::move(std::begin(rhs), std::end(rhs), std::back_inserter(lhs)); + return lhs; +} diff --git a/ext/librethinkdbxx/test/testlib.h b/ext/librethinkdbxx/test/testlib.h new file mode 100644 index 00000000..5b795436 --- /dev/null +++ b/ext/librethinkdbxx/test/testlib.h @@ -0,0 +1,231 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace R = RethinkDB; + +extern std::vector> section; +extern int failed; +extern int count; +extern std::unique_ptr conn; +extern int verbosity; + +const char* indent(); + +void enter_section(const char* name); +void section_cleanup(); +void exit_section(); + +#define TEST_DO(code) \ + if (verbosity > 1) fprintf(stderr, "%sTEST: %s\n", indent(), #code); \ + code + +#define TEST_EQ(code, expected) \ + do { \ + if (verbosity > 1) fprintf(stderr, "%sTEST: %s\n", indent(), #code); \ + try { test_eq(#code, (code), (expected)); } \ + catch (const R::Error& error) { test_eq(#code, error, (expected)); } \ + } while (0) + +struct err { + err(const char* type_, std::string message_, R::Array&& backtrace_ = {}) : + type(type_), message(message_), backtrace(std::move(backtrace_)) { } + + std::string convert_type() const { + return type; + } + + static std::string trim_message(std::string msg) { + size_t i = msg.find(":\n"); + if (i != std::string::npos) { + return msg.substr(0, i + 1); + } + return msg; + } + + std::string type; + std::string message; + R::Array backtrace; +}; + +struct err_regex { + err_regex(const char* type_, const char* message_, R::Array&& backtrace_ = {}) : + type(type_), message(message_), backtrace(std::move(backtrace_)) { } + std::string type; + std::string message; + R::Array backtrace; + std::string regex() const { + return type + ": " + message; + } +}; + +R::Object regex(const char* pattern); + +bool match(const char* pattern, const char* string); + +R::Object partial(R::Object&& object); +R::Object partial(R::Array&& array); +R::Datum uuid(); +R::Object arrlen(int n, R::Datum&& datum); +R::Object arrlen(int n); +R::Term new_table(); +std::string repeat(std::string&& s, int n); +R::Term fetch(R::Cursor& cursor, int count = -1, double timeout = 1); +R::Object bag(R::Array&& array); +R::Object bag(R::Datum&& d); + +struct temp_table { + temp_table() { + char chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + char name_[15] = "temp_"; + for (unsigned int i = 5; i + 1 < sizeof name_; ++i) { + name_[i] = chars[random() % (sizeof chars - 1)]; + } + name_[14] = 0; + R::table_create(name_).run(*conn); + name = name_; + } + + ~temp_table() { + try { + R::table_drop(name).run(*conn); + } catch (const R::Error &e) { + if(!strstr(e.message.c_str(), "does not exist")){ + printf("error dropping temp_table: %s\n", e.message.c_str()); + } + } + } + + R::Term table() { return R::table(name); } + std::string name; +}; + +void clean_slate(); + +// std::string to_string(const R::Cursor&); +std::string to_string(const R::Term&); +std::string to_string(const R::Datum&); +std::string to_string(const R::Error&); +std::string to_string(const err_regex&); +std::string to_string(const err&); + +bool equal(const R::Datum&, const R::Datum&); +bool equal(const R::Error&, const err_regex&); +bool equal(const R::Error&, const err&); + +template +bool equal(const T& a, const err& b) { + return false; +} + +template +bool equal(const T& a, const err_regex& b) { + return false; +} + +template +bool equal(const R::Error& a, const T& b) { + return false; +} + +std::string truncate(std::string&&); + +template +void test_eq(const char* code, const T val, const U expected) { + + try { + count ++; + if (!equal(val, expected)) { + failed++; + for (auto& it : section) { + if (it.second) { + printf("%sSection: %s\n", indent(), it.first); + it.second = false; + } + } + try { + printf("%sFAILURE in ‘%s’:\n%s Expected: ‘%s’\n%s but got: ‘%s’\n", + indent(), code, + indent(), truncate(to_string(expected)).c_str(), + indent(), truncate(to_string(val)).c_str()); + } catch (const R::Error& e) { + printf("%sFAILURE: Failed to print failure description: %s\n", indent(), e.message.c_str()); + } catch (...) { + printf("%sFAILURE: Failed to print failure description\n", indent()); + } + } + } catch (const std::regex_error& rx_err) { + printf("%sSKIP: error with regex (likely a buggy regex implementation): %s\n", indent(), rx_err.what()); + } +} + +template +void test_eq(const char* code, const R::Cursor& val, const U expected) { + try { + R::Datum result = val.to_datum(); + test_eq(code, result, expected); + } catch (R::Error& error) { + test_eq(code, error, expected); + } +} + +int len(const R::Datum&); + +R::Term wait(int n); + +#define PacificTimeZone() (-7 * 3600) +#define UTCTimeZone() (0) + +extern R::Datum nil; + +inline R::Cursor maybe_run(R::Cursor& c, R::Connection&, R::OptArgs&& o = {}) { + return std::move(c); +} + +inline R::Cursor maybe_run(R::Term q, R::Connection& c, R::OptArgs&& o = {}) { + return q.run(c, std::move(o)); +} + +inline int operator+(R::Datum a, int b) { + return a.extract_number() + b; +} + +inline R::Array operator*(R::Array arr, int n) { + R::Array ret; + for(int i = 0; i < n; i++) { + for(const auto& it: arr) { + ret.push_back(it); + } + } + return ret; +} + +inline R::Array array_range(int x, int y) { + R::Array ret; + for(int i = x; i < y; ++i) { + ret.push_back(i); + } + return ret; +} + +template +inline R::Array array_map(F f, R::Array a){ + R::Array ret; + for(R::Datum& d: a) { + ret.push_back(f(d.extract_number())); + } + return ret; +} + +R::Array append(R::Array lhs, R::Array rhs); + +template +std::string str(T x){ + return to_string(x); +} diff --git a/ext/librethinkdbxx/test/upstream/aggregation.yaml b/ext/librethinkdbxx/test/upstream/aggregation.yaml new file mode 100644 index 00000000..1e0ec9c8 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/aggregation.yaml @@ -0,0 +1,575 @@ +desc: Tests that manipulation data in tables +table_variable_name: tbl tbl2 tbl3 tbl4 +tests: + + # Set up some data + - cd: r.range(100).for_each(tbl.insert({'id':r.row, 'a':r.row.mod(4)})) + rb: tbl.insert((0..99).map{ |i| { :id => i, :a => i % 4 } }) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100} + + - cd: r.range(100).for_each(tbl2.insert({'id':r.row, 'a':r.row.mod(4)})) + rb: tbl2.insert((0..99).map{ |i| { :id => i, :b => i % 4 } }) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100} + + - cd: r.range(100).for_each(tbl3.insert({'id':r.row, 'a':r.row.mod(4), 'b':{'c':r.row.mod(5)}})) + rb: tbl3.insert((0..99).map{ |i| { :id => i, :a => i % 4, :b => { :c => i % 5 } } }) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100} + + - def: + cd: time1 = 1375115782.24 + js: time1 = 1375115782.24 * 1000 + + - def: + cd: time2 = 1375147296.68 + js: time2 = 1375147296.68 * 1000 + + - cd: + - tbl4.insert({'id':0, 'time':r.epoch_time(time1)}) + - tbl4.insert({'id':1, 'time':r.epoch_time(time2)}) + ot: {'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':1} + + # GMR + + - cd: tbl.sum('a') + ot: 150 + - rb: tbl.map{|row| row['a']}.sum() + py: tbl.map(lambda row:row['a']).sum() + js: tbl.map(function(row){return row('a')}).sum() + ot: 150 + - cd: tbl.group('a').sum('id') + ot: + cd: ({0:1200, 1:1225, 2:1250, 3:1275}) + js: ([{'group':0,'reduction':1200},{'group':1,'reduction':1225},{'group':2,'reduction':1250},{'group':3,'reduction':1275}]) + - cd: tbl.avg('a') + ot: 1.5 + - rb: tbl.map{|row| row['a']}.avg() + py: tbl.map(lambda row:row['a']).avg() + js: tbl.map(function(row){return row('a')}).avg() + ot: 1.5 + - cd: tbl.group('a').avg('id') + ot: + cd: {0:48, 1:49, 2:50, 3:51} + js: [{'group':0,'reduction':48},{'group':1,'reduction':49},{'group':2,'reduction':50},{'group':3,'reduction':51}] + - cd: tbl.min('a')['a'] + js: tbl.min('a')('a') + ot: 0 + - cd: tbl.order_by('id').min('a') + ot: {'a':0, 'id':0} + - rb: tbl.map{|row| row['a']}.min() + py: tbl.map(lambda row:row['a']).min() + js: tbl.map(function(row){return row('a')}).min() + ot: 0 + - cd: tbl.group('a').min('id') + ot: + cd: {0:{'a':0, 'id':0}, 1:{'a':1, 'id':1}, 2:{'a':2, 'id':2}, 3:{'a':3, 'id':3}} + js: [{'group':0,'reduction':{'a':0, 'id':0}},{'group':1,'reduction':{'a':1, 'id':1}},{'group':2,'reduction':{'a':2, 'id':2}},{'group':3,'reduction':{'a':3, 'id':3}}] + - cd: tbl.order_by('id').max('a') + ot: {'a':3, 'id':3} + - rb: tbl.map{|row| row['a']}.max() + py: tbl.map(lambda row:row['a']).max() + js: tbl.map(function(row){return row('a')}).max() + ot: 3 + - cd: tbl.group('a').max('id') + ot: + cd: {0:{'a':0, 'id':96}, 1:{'a':1, 'id':97}, 2:{'a':2, 'id':98}, 3:{'a':3, 'id':99}} + js: [{'group':0,'reduction':{'a':0, 'id':96}},{'group':1,'reduction':{'a':1, 'id':97}},{'group':2,'reduction':{'a':2, 'id':98}},{'group':3,'reduction':{'a':3, 'id':99}}] + + - cd: tbl.min() + ot: {"a":0, "id":0} + - cd: tbl.group('a').min() + ot: + cd: {0:{"a":0, "id":0}, 1:{"a":1, "id":1}, 2:{"a":2, "id":2}, 3:{"a":3, "id":3}} + js: [{'group':0,'reduction':{"a":0,"id":0}},{'group':1,'reduction':{"a":1,"id":1}},{'group':2,'reduction':{"a":2,"id":2}},{'group':3,'reduction':{"a":3,"id":3}}] + - cd: tbl.max() + ot: {"a":3, "id":99} + - cd: tbl.group('a').max() + ot: + cd: {0:{'a':0, 'id':96}, 1:{'a':1, 'id':97}, 2:{'a':2, 'id':98}, 3:{'a':3, 'id':99}} + js: [{'group':0,'reduction':{"a":0,"id":96}},{'group':1,'reduction':{"a":1,"id":97}},{'group':2,'reduction':{"a":2,"id":98}},{'group':3,'reduction':{"a":3,"id":99}}] + + - rb: tbl.sum{|row| row['a']} + py: + - tbl.sum(lambda row:row['a']) + - tbl.sum(r.row['a']) + js: + - tbl.sum(function(row){return row('a')}) + - tbl.sum(r.row('a')) + ot: 150 + - rb: tbl.map{|row| row['a']}.sum() + py: tbl.map(lambda row:row['a']).sum() + js: tbl.map(function(row){return row('a')}).sum() + ot: 150 + - rb: tbl.group{|row| row['a']}.sum{|row| row['id']} + py: tbl.group(lambda row:row['a']).sum(lambda row:row['id']) + js: tbl.group(function(row){return row('a')}).sum(function(row){return row('id')}) + ot: + cd: {0:1200, 1:1225, 2:1250, 3:1275} + js: [{'group':0,'reduction':1200},{'group':1,'reduction':1225},{'group':2,'reduction':1250},{'group':3,'reduction':1275}] + - rb: + - tbl.avg{|row| row['a']} + py: + - tbl.avg(lambda row:row['a']) + - tbl.avg(r.row['a']) + js: + - tbl.avg(function(row){return row('a')}) + - tbl.avg(r.row('a')) + ot: 1.5 + - rb: tbl.map{|row| row['a']}.avg() + py: tbl.map(lambda row:row['a']).avg() + js: tbl.map(function(row){return row('a')}).avg() + ot: 1.5 + - rb: tbl.group{|row| row['a']}.avg{|row| row['id']} + py: tbl.group(lambda row:row['a']).avg(lambda row:row['id']) + js: tbl.group(function(row){return row('a')}).avg(function(row){return row('id')}) + ot: + cd: {0:48, 1:49, 2:50, 3:51} + js: [{'group':0,'reduction':48},{'group':1,'reduction':49},{'group':2,'reduction':50},{'group':3,'reduction':51}] + - rb: tbl.order_by(r.desc('id')).min{|row| row['a']} + py: + - tbl.order_by(r.desc('id')).min(lambda row:row['a']) + - tbl.order_by(r.desc('id')).min(r.row['a']) + js: + - tbl.order_by(r.desc('id')).min(function(row){return row('a')}) + - tbl.order_by(r.desc('id')).min(r.row('a')) + ot: {'a':0, 'id':96} + - rb: + - tbl.order_by(r.desc('id')).min{|row| row['a']}['a'] + py: + - tbl.order_by(r.desc('id')).min(lambda row:row['a'])['a'] + - tbl.order_by(r.desc('id')).min(r.row['a'])['a'] + js: + - tbl.order_by(r.desc('id')).min(function(row){return row('a')})('a') + - tbl.order_by(r.desc('id')).min(r.row('a'))('a') + ot: 0 + - rb: tbl.map{|row| row['a']}.min() + py: tbl.map(lambda row:row['a']).min() + js: tbl.map(function(row){return row('a')}).min() + ot: 0 + - rb: tbl.group{|row| row['a']}.min{|row| row['id']}['id'] + py: tbl.group(lambda row:row['a']).min(lambda row:row['id'])['id'] + js: tbl.group(function(row){return row('a')}).min(function(row){return row('id')})('id') + ot: + cd: {0:0, 1:1, 2:2, 3:3} + js: [{'group':0,'reduction':0},{'group':1,'reduction':1},{'group':2,'reduction':2},{'group':3,'reduction':3}] + - rb: + - tbl.max{|row| row['a']}['a'] + py: + - tbl.max(lambda row:row['a'])['a'] + - tbl.max(r.row['a'])['a'] + js: + - tbl.max(function(row){return row('a')})('a') + - tbl.max(r.row('a'))('a') + ot: 3 + - rb: tbl.map{|row| row['a']}.max() + py: tbl.map(lambda row:row['a']).max() + js: tbl.map(function(row){return row('a')}).max() + ot: 3 + - rb: tbl.group{|row| row['a']}.max{|row| row['id']}['id'] + py: tbl.group(lambda row:row['a']).max(lambda row:row['id'])['id'] + js: tbl.group(function(row){return row('a')}).max(function(row){return row('id')})('id') + ot: + cd: {0:96, 1:97, 2:98, 3:99} + js: [{'group':0,'reduction':96},{'group':1,'reduction':97},{'group':2,'reduction':98},{'group':3,'reduction':99}] + + - rb: tbl.group{|row| row[:a]}.map{|row| row[:id]}.reduce{|a,b| a+b} + py: tbl.group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a+b) + js: tbl.group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + ot: + cd: {0:1200, 1:1225, 2:1250, 3:1275} + js: [{'group':0,'reduction':1200},{'group':1,'reduction':1225},{'group':2,'reduction':1250},{'group':3,'reduction':1275}] + + - rb: tbl.group{|row| row[:a]}.map{|row| row[:id]}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a+b) + - tbl.group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 1200], [1, 1225], [2, 1250], [3, 1275]]} + + - cd: r.expr([{'a':1}]).filter(true).limit(1).group('a') + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[1, [{'a':1}]]]} + + # GMR + - cd: tbl.group('a').type_of() + ot: "GROUPED_STREAM" + - cd: tbl.group('a').count().type_of() + ot: "GROUPED_DATA" + - cd: tbl.group('a').coerce_to('ARRAY').type_of() + ot: "GROUPED_DATA" + + - rb: tbl.orderby(index:'id').filter{|row| row['id'].lt(10)}.group('a').map{|row| row['id']}.coerce_to('ARRAY') + py: tbl.order_by(index='id').filter(lambda row:row['id'] < 10).group('a').map(lambda row:row['id']).coerce_to('ARRAY') + js: tbl.orderBy({index:'id'}).filter(function(row){return row('id').lt(10)}).group('a').map(function(row){return row('id')}).coerce_to('ARRAY') + ot: + cd: {0:[0,4,8],1:[1,5,9],2:[2,6],3:[3,7]} + js: [{'group':0,'reduction':[0,4,8]},{'group':1,'reduction':[1,5,9]},{'group':2,'reduction':[2,6]},{'group':3,'reduction':[3,7]}] + + - rb: tbl.filter{|row| row['id'].lt(10)}.group('a').count().do{|x| x*x} + py: tbl.filter(lambda row:row['id'] < 10).group('a').count().do(lambda x:x*x) + js: tbl.filter(function(row){return row('id').lt(10)}).group('a').count().do(function(x){return x.mul(x)}) + ot: + cd: {0:9,1:9,2:4,3:4} + js: [{'group':0,'reduction':9},{'group':1,'reduction':9},{'group':2,'reduction':4},{'group':3,'reduction':4}] + + - rb: tbl.union(tbl).group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.union(tbl).group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.union(tbl).group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.union(tbl).group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.union(tbl).group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 2400], [1, 2450], [2, 2500], [3, 2550]]} + + # GMR + - rb: tbl.coerce_to("array").union(tbl).group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.coerce_to("array").union(tbl).group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.coerce_to("array").union(tbl).group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.coerce_to("array").union(tbl).group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.coerce_to("array").union(tbl).group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 2400], [1, 2450], [2, 2500], [3, 2550]]} + + # GMR + - rb: tbl.union(tbl.coerce_to("array")).group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + py: + - tbl.union(tbl.coerce_to("array")).group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.union(tbl.coerce_to("array")).group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.union(tbl.coerce_to("array")).group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.union(tbl.coerce_to("array")).group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 2400], [1, 2450], [2, 2500], [3, 2550]]} + + - py: + - tbl.group(lambda row:row['a']).map(lambda row:row['id']).reduce(lambda a,b:a + b) + - tbl.group(r.row['a']).map(r.row['id']).reduce(lambda a,b:a + b) + js: + - tbl.group(function(row){return row('a')}).map(function(row){return row('id')}).reduce(function(a,b){return a.add(b)}) + - tbl.group(r.row('a')).map(r.row('id')).reduce(function(a,b){return a.add(b)}) + - tbl.group('a').map(r.row('id')).reduce(function(a,b){return a.add(b)}) + rb: tbl.group('a').map{|x| x['id']}.reduce{|a,b| a+b} + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 1200], [1, 1225], [2, 1250], [3, 1275]]} + + # undefined... + - js: + - tbl.group(function(row){}) + - tbl.map(function(row){}) + - tbl.reduce(function(row){}) + - tbl.group(r.row('a')).group(function(row){}) + - tbl.group(r.row('a')).map(function(row){}) + - tbl.group(r.row('a')).reduce(function(row){}) + - tbl.map(r.row('id')).group(function(row){}) + - tbl.map(r.row('id')).map(function(row){}) + - tbl.map(r.row('id')).reduce(function(row){}) + - tbl.reduce(function(a,b){return a+b}).group(function(row){}) + - tbl.reduce(function(a,b){return a+b}).map(function(row){}) + - tbl.reduce(function(a,b){return a+b}).reduce(function(row){}) + ot: err('ReqlDriverCompileError', 'Anonymous function returned `undefined`. Did you forget a `return`?', [0]) + + # GroupBy + + # COUNT + + - cd: tbl.group('a').count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 25], [1, 25], [2, 25], [3, 25]]} + + # SUM + - cd: tbl.group('a').sum('id') + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 1200], [1, 1225], [2, 1250], [3, 1275]]} + + # AVG + - cd: tbl.group('a').avg('id') + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 48], [1, 49], [2, 50], [3, 51]]} + + # Pattern Matching + - rb: tbl3.group{|row| row['b']['c']}.count() + py: tbl3.group(lambda row:row['b']['c']).count() + js: tbl3.group(function(row){return row('b')('c')}).count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 20], [1, 20], [2, 20], [3, 20], [4, 20]]} + + # Multiple keys + - rb: tbl.group('a', lambda {|row| row['id']%3}).count() + py: tbl.group('a', lambda row:row['id'].mod(3)).count() + js: tbl.group('a', function(row){return row('id').mod(3)}).count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, 0], 9], [[0, 1], 8], [[0, 2], 8], [[1, 0], 8], [[1, 1], 9], [[1, 2], 8], [[2, 0], 8], [[2, 1], 8], [[2, 2], 9], [[3, 0], 9], [[3, 1], 8], [[3, 2], 8]]} + + # Grouping by time + - rb: tbl4.group('time').coerce_to('array') + runopts: + time_format: 'raw' + ot: + rb: {{"$reql_type$":"TIME","epoch_time":1375115782.24,"timezone":"+00:00"}:[{"id":0,"time":{"$reql_type$":"TIME","epoch_time":1375115782.24,"timezone":"+00:00"}}],{"$reql_type$":"TIME","epoch_time":1375147296.68,"timezone":"+00:00"}:[{"id":1,"time":{"$reql_type$":"TIME","epoch_time":1375147296.68,"timezone":"+00:00"}}]} + py: {frozenset([('$reql_type$','TIME'),('timezone','+00:00'),('epoch_time',1375115782.24)]):[{'id':0,'time':{'timezone':'+00:00','$reql_type$':'TIME','epoch_time':1375115782.24}}],frozenset([('$reql_type$','TIME'),('timezone','+00:00'),('epoch_time',1375147296.68)]):[{'id':1,'time':{'timezone':'+00:00','$reql_type$':'TIME','epoch_time':1375147296.68}}]} + js: [{'group':{"$reql_type$":"TIME","epoch_time":1375115782240,"timezone":"+00:00"},'reduction':[{"id":0,"time":{"$reql_type$":"TIME","epoch_time":1375115782240,"timezone":"+00:00"}}]},{'group':{"$reql_type$":"TIME","epoch_time":1375147296680,"timezone":"+00:00"},'reduction':[{"id":1,"time":{"$reql_type$":"TIME","epoch_time":1375147296680,"timezone":"+00:00"}}]}] + + # Distinct + - py: tbl.map(lambda row:row['a']).distinct().count() + js: tbl.map(function(row) { return row('a'); }).distinct().count() + rb: tbl.map{ |row| row[:a] }.distinct.count + ot: 4 + + - cd: tbl.distinct().type_of() + ot: "STREAM" + + - cd: tbl.distinct().count() + ot: 100 + + - cd: tbl.distinct({index:'id'}).type_of() + py: tbl.distinct(index='id').type_of() + ot: "STREAM" + + - cd: tbl.distinct({index:'id'}).count() + py: tbl.distinct(index='id').count() + ot: 100 + + - cd: tbl.index_create('a') + ot: {'created':1} + + - rb: tbl.index_create('m', multi:true){|row| [row['a'], row['a']]} + ot: {'created':1} + + - rb: tbl.index_create('m2', multi:true){|row| [1, 2]} + ot: {'created':1} + + - cd: tbl.index_wait('a').pluck('index', 'ready') + ot: [{'index':'a','ready':true}] + + - rb: tbl.index_wait('m').pluck('index', 'ready') + ot: [{'index':'m','ready':true}] + + - rb: tbl.index_wait('m2').pluck('index', 'ready') + ot: [{'index':'m2','ready':true}] + + - cd: tbl.between(0, 1, {index:'a'}).distinct().count() + py: tbl.between(0, 1, index='a').distinct().count() + ot: 25 + + - cd: tbl.between(0, 1, {index:'a'}).distinct({index:'id'}).count() + py: tbl.between(0, 1, index='a').distinct(index='id').count() + ot: 25 + + - rb: tbl.between(0, 1, {index:'m'}).count() + ot: 50 + + - rb: tbl.between(0, 1, {index:'m'}).distinct().count() + ot: 25 + + - rb: tbl.orderby({index:'m'}).count() + ot: 200 + + - rb: tbl.orderby({index:'m'}).distinct().count() + ot: 100 + + - rb: tbl.orderby({index:r.desc('m')}).count() + ot: 200 + + - rb: tbl.orderby({index:r.desc('m')}).distinct().count() + ot: 100 + + - rb: tbl.between(1, 3, {index:'m2'}).count() + ot: 200 + + - rb: tbl.between(1, 3, {index:'m2'}).distinct().count() + ot: 100 + + - rb: tbl.between(1, 3, {index:'m2'}).orderby(index:r.desc('m2')).distinct().count() + ot: 100 + + - rb: tbl.between(0, 1, {index:'m'}).count() + ot: 50 + + - rb: tbl.between(0, 1, {index:'m'}).distinct().count() + ot: 25 + + - cd: tbl.distinct({index:'a'}).type_of() + py: tbl.distinct(index='a').type_of() + ot: "STREAM" + + - cd: tbl.distinct({index:'a'}).count() + py: tbl.distinct(index='a').count() + ot: 4 + + - cd: tbl.group() + ot: err('ReqlQueryLogicError', 'Cannot group by nothing.', []) + + - py: tbl.group(index='id').count() + js: tbl.group({index:'id'}).count() + cd: tbl.group(index:'id').count + runopts: + group_format: 'raw' + ot: ({'$reql_type$':'GROUPED_DATA', 'data':[[0, 1], [1, 1], [2, 1], [3, 1], [4, 1], [5, 1], [6, 1], [7, 1], [8, 1], [9, 1], [10, 1], [11, 1], [12, 1], [13, 1], [14, 1], [15, 1], [16, 1], [17, 1], [18, 1], [19, 1], [20, 1], [21, 1], [22, 1], [23, 1], [24, 1], [25, 1], [26, 1], [27, 1], [28, 1], [29, 1], [30, 1], [31, 1], [32, 1], [33, 1], [34, 1], [35, 1], [36, 1], [37, 1], [38, 1], [39, 1], [40, 1], [41, 1], [42, 1], [43, 1], [44, 1], [45, 1], [46, 1], [47, 1], [48, 1], [49, 1], [50, 1], [51, 1], [52, 1], [53, 1], [54, 1], [55, 1], [56, 1], [57, 1], [58, 1], [59, 1], [60, 1], [61, 1], [62, 1], [63, 1], [64, 1], [65, 1], [66, 1], [67, 1], [68, 1], [69, 1], [70, 1], [71, 1], [72, 1], [73, 1], [74, 1], [75, 1], [76, 1], [77, 1], [78, 1], [79, 1], [80, 1], [81, 1], [82, 1], [83, 1], [84, 1], [85, 1], [86, 1], [87, 1], [88, 1], [89, 1], [90, 1], [91, 1], [92, 1], [93, 1], [94, 1], [95, 1], [96, 1], [97, 1], [98, 1], [99, 1]]}) + + - py: tbl.group(index='a').count() + js: tbl.group({index:'a'}).count() + rb: tbl.group(index:'a').count + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[0, 25], [1, 25], [2, 25], [3, 25]]} + + - py: tbl.group('a', index='id').count() + js: tbl.group('a', {index:'id'}).count() + rb: tbl.group('a', index:'id').count + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, 0], 1], [[0, 4], 1], [[0, 8], 1], [[0, 12], 1], [[0, 16], 1], [[0, 20], 1], [[0, 24], 1], [[0, 28], 1], [[0, 32], 1], [[0, 36], 1], [[0, 40], 1], [[0, 44], 1], [[0, 48], 1], [[0, 52], 1], [[0, 56], 1], [[0, 60], 1], [[0, 64], 1], [[0, 68], 1], [[0, 72], 1], [[0, 76], 1], [[0, 80], 1], [[0, 84], 1], [[0, 88], 1], [[0, 92], 1], [[0, 96], 1], [[1, 1], 1], [[1, 5], 1], [[1, 9], 1], [[1, 13], 1], [[1, 17], 1], [[1, 21], 1], [[1, 25], 1], [[1, 29], 1], [[1, 33], 1], [[1, 37], 1], [[1, 41], 1], [[1, 45], 1], [[1, 49], 1], [[1, 53], 1], [[1, 57], 1], [[1, 61], 1], [[1, 65], 1], [[1, 69], 1], [[1, 73], 1], [[1, 77], 1], [[1, 81], 1], [[1, 85], 1], [[1, 89], 1], [[1, 93], 1], [[1, 97], 1], [[2, 2], 1], [[2, 6], 1], [[2, 10], 1], [[2, 14], 1], [[2, 18], 1], [[2, 22], 1], [[2, 26], 1], [[2, 30], 1], [[2, 34], 1], [[2, 38], 1], [[2, 42], 1], [[2, 46], 1], [[2, 50], 1], [[2, 54], 1], [[2, 58], 1], [[2, 62], 1], [[2, 66], 1], [[2, 70], 1], [[2, 74], 1], [[2, 78], 1], [[2, 82], 1], [[2, 86], 1], [[2, 90], 1], [[2, 94], 1], [[2, 98], 1], [[3, 3], 1], [[3, 7], 1], [[3, 11], 1], [[3, 15], 1], [[3, 19], 1], [[3, 23], 1], [[3, 27], 1], [[3, 31], 1], [[3, 35], 1], [[3, 39], 1], [[3, 43], 1], [[3, 47], 1], [[3, 51], 1], [[3, 55], 1], [[3, 59], 1], [[3, 63], 1], [[3, 67], 1], [[3, 71], 1], [[3, 75], 1], [[3, 79], 1], [[3, 83], 1], [[3, 87], 1], [[3, 91], 1], [[3, 95], 1], [[3, 99], 1]]} + + - py: tbl.group('a', index='a').count() + js: tbl.group('a', {index:'a'}).count() + rb: tbl.group('a', index:'a').count + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, 0], 25], [[1, 1], 25], [[2, 2], 25], [[3, 3], 25]]} + + - rb: tbl.group('a', lambda {|row| 'f'}, lambda {|row| []}, lambda {|row| [{}, [0], null, 0]}, multi:true).count + py: tbl.group('a', lambda row:'f', lambda row:[], lambda row:[{}, [0], null, 0], multi=True).count() + js: tbl.group('a', function(row){return 'f';}, function(row){return [];}, function(row){return [{}, [0], null, 0];}, {multi:true}).count() + runopts: + group_format: 'raw' + ot: {'$reql_type$':'GROUPED_DATA', 'data':[[[0, "f", null, [0]], 25], [[0, "f", null, null], 25], [[0, "f", null, 0], 25], [[0, "f", null, {}], 25], [[1, "f", null, [0]], 25], [[1, "f", null, null], 25], [[1, "f", null, 0], 25], [[1, "f", null, {}], 25], [[2, "f", null, [0]], 25], [[2, "f", null, null], 25], [[2, "f", null, 0], 25], [[2, "f", null, {}], 25], [[3, "f", null, [0]], 25], [[3, "f", null, null], 25], [[3, "f", null, 0], 25], [[3, "f", null, {}], 25]]} + + - cd: tbl.group('a').count().ungroup() + ot: [{'group':0, 'reduction':25}, {'group':1, 'reduction':25}, {'group':2, 'reduction':25}, {'group':3, 'reduction':25}] + + - cd: tbl.group('a').ungroup()['group'] + js: tbl.group('a').ungroup()('group') + ot: [0, 1, 2, 3] + + - py: tbl.order_by(index='id').limit(16).group('a','a').map(r.row['id']).sum().ungroup() + js: tbl.order_by({index:'id'}).limit(16).group('a','a').map(r.row('id')).sum().ungroup() + rb: tbl.order_by(index:'id').limit(16).group('a','a').map{|row| row['id']}.sum().ungroup() + ot: [{'group':[0,0],'reduction':24},{'group':[1,1],'reduction':28},{'group':[2,2],'reduction':32},{'group':[3,3],'reduction':36}] + + - cd: tbl.group('a', null).count().ungroup() + ot: [{'group':[0,null],'reduction':25},{'group':[1,null],'reduction':25},{'group':[2,null],'reduction':25},{'group':[3,null],'reduction':25}] + + - py: tbl.group('a', lambda row:[1,'two'], multi=True).count().ungroup() + js: tbl.group('a', function(row){return [1,'two']},{multi:true}).count().ungroup() + rb: tbl.group('a', lambda {|row| [1,'two']}, multi:true).count().ungroup() + ot: [{'group':[0,1],'reduction':25},{'group':[0,'two'],'reduction':25},{'group':[1,1],'reduction':25},{'group':[1,'two'],'reduction':25},{'group':[2,1],'reduction':25},{'group':[2,'two'],'reduction':25},{'group':[3,1],'reduction':25},{'group':[3,'two'],'reduction':25}] + + # proper test for seq.count() + - cd: tbl.count() + ot: 100 + + - js: tbl.filter(r.row('a').ne(1).and(r.row('id').gt(10))).update({'b':r.row('a').mul(10)}) + py: tbl.filter(r.row['a'].ne(1).and_(r.row['id'].gt(10))).update({'b':r.row['a'] * 10}) + rb: tbl.filter{|row| row['a'].ne(1).and(row['id'].gt(10))}.update{|row| {'b'=>row['a'] * 10}} + ot: partial({'errors':0, 'replaced':67}) + + - cd: tbl.group('b').count() + ot: + cd: {null:33, 0:22, 20:22, 30:23} + js: [{"group":null, "reduction":33}, {"group":0, "reduction":22}, {"group":20, "reduction":22}, {"group":30, "reduction":23}] + + - cd: tbl.group('a').sum('b') + ot: + cd: {0:0, 2:440, 3:690} + js: [{"group":0, "reduction":0}, {"group":2, "reduction":440}, {"group":3, "reduction":690}] + + - cd: tbl.group('a').avg('b') + ot: + cd: {0:0, 2:20, 3:30} + js: [{"group":0, "reduction":0}, {"group":2, "reduction":20}, {"group":3, "reduction":30}] + + - cd: tbl.order_by('id').group('a').min('b') + ot: + cd: {0:{"a":0, "b":0, "id":12}, 2:{"a":2, "b":20, "id":14}, 3:{"a":3, "b":30, "id":11}} + js: [{"group":0, "reduction":{"a":0, "b":0, "id":12}}, {"group":2, "reduction":{"a":2, "b":20, "id":14}}, {"group":3, "reduction":{"a":3, "b":30, "id":11}}] + + - cd: tbl.order_by('id').group('a').min('id') + ot: + cd: {0:{"a":0, "id":0}, 1:{"a":1, "id":1}, 2:{"a":2, "id":2}, 3:{"a":3, "id":3}} + js: [{"group":0, "reduction":{"a":0, "id":0}}, {"group":1, "reduction":{"a":1, "id":1}}, {"group":2, "reduction":{"a":2, "id":2}}, {"group":3, "reduction":{"a":3, "id":3}}] + + - cd: tbl.order_by('id').group('a').max('b') + ot: + cd: {0:{"a":0, "b":0, "id":12}, 2:{"a":2, "b":20, "id":14}, 3:{"a":3, "b":30, "id":11}} + js: [{"group":0, "reduction":{"a":0,"b":0, "id":12}}, {"group":2, "reduction":{"a":2, "b":20, "id":14}}, {"group":3, "reduction":{"a":3, "b":30, "id":11}}] + + - cd: tbl.min() + ot: {'a':0,'id':0} + - py: tbl.min(index='id') + rb: tbl.min(index:'id') + js: tbl.min({index:'id'}) + ot: {'a':0,'id':0} + - py: tbl.min(index='a') + rb: tbl.min(index:'a') + js: tbl.min({index:'a'}) + ot: {'a':0,'id':0} + + - cd: tbl.max().without('b') + ot: {'a':3,'id':99} + - py: tbl.max(index='id').without('b') + rb: tbl.max(index:'id').without('b') + js: tbl.max({index:'id'}).without('b') + ot: {'a':3,'id':99} + - py: tbl.max(index='a').without('b') + rb: tbl.max(index:'a').without('b') + js: tbl.max({index:'a'}).without('b') + ot: {'a':3,'id':99} + + + # Infix + + - cd: r.group([ 1, 1, 2 ], r.row).count().ungroup() + rb: r.group([ 1, 1, 2 ]) {|row| row}.count().ungroup() + ot: [ {'group': 1, 'reduction': 2}, {'group': 2, 'reduction': 1} ] + - cd: + - r.count([ 1, 2 ]) + - r.count([ 1, 2 ], r.row.gt(0)) + rb: + - r.count([ 1, 2 ]) + - r.count([ 1, 2 ]) {|row| row.gt(0)} + ot: 2 + - cd: + - r.sum([ 1, 2 ]) + - r.sum([ 1, 2 ], r.row) + rb: r.sum([ 1, 2 ]) + ot: 3 + - cd: + - r.avg([ 1, 2 ]) + - r.avg([ 1, 2 ], r.row) + rb: r.avg([ 1, 2 ]) + ot: 1.5 + - cd: + - r.min([ 1, 2 ]) + - r.min([ 1, 2 ], r.row) + rb: r.min([ 1, 2 ]) + ot: 1 + - cd: + - r.max([ 1, 2 ]) + - r.max([ 1, 2 ], r.row) + rb: r.max([ 1, 2 ]) + ot: 2 + - cd: r.distinct([ 1, 1 ]) + ot: [ 1 ] + - cd: + - r.contains([ 1, 2 ]) + - r.contains([ 1, 2 ], r.row.gt(0)) + rb: + - r.contains([ 1, 2 ]) + - r.contains([ 1, 2 ]) {|row| row.gt(0)} + ot: true diff --git a/ext/librethinkdbxx/test/upstream/arity.yaml b/ext/librethinkdbxx/test/upstream/arity.yaml new file mode 100644 index 00000000..8197fe44 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/arity.yaml @@ -0,0 +1,316 @@ +desc: Test the arity of every function +table_variable_name: tbl +tests: + + # TODO: add test for slice (should require one or two arguments) + + # Set up some data + - def: db = r.db('test') + - def: obj = r.expr({'a':1}) + - def: array = r.expr([1]) + + - ot: err("ReqlCompileError", "Expected 0 arguments but found 1.", []) + cd: r.db_list(1) + + - ot: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + cd: + - tbl.zip(1) + - tbl.is_empty(1) + - obj.keys(1) + + - cd: tbl.distinct(1) + ot: + cd: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + js: err("ReqlCompileError", "Expected 0 arguments (not including options) but found 1.", []) + + - cd: tbl.delete(1) + ot: + js: err("ReqlCompileError", "Expected 0 arguments (not including options) but found 1.", []) + cd: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + + - rb: db.table_list(1) + ot: err("ReqlCompileError", "Expected between 0 and 1 arguments but found 2.", []) + + - ot: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + cd: + - r.db_create() + - r.db_drop() + - r.db() + - r.floor() + - r.ceil() + - r.round() + + - cd: r.error() + ot: err("ReqlQueryLogicError", "Empty ERROR term outside a default block.", []) + + - cd: r.js() + ot: + cd: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + + - cd: r.expr() + ot: + py3.3: err_regex('TypeError', '.* missing 1 required positional argument.*', []) + py3.4: err_regex('TypeError', '.* missing 1 required positional argument.*', []) + py3.5: err_regex('TypeError', '.* missing 1 required positional argument.*', []) + py: err_regex('TypeError', ".* takes at least 1 (?:positional )?argument \(0 given\)", []) + js: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 0.", []) + rb: err("ArgumentError", 'wrong number of arguments (0 for 1)', []) + rb2: err("ArgumentError", 'wrong number of arguments (0 for 1..2)', []) + + - ot: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + cd: + - tbl.concat_map() + - tbl.skip() + - tbl.limit() + - array.append() + - array.prepend() + - array.difference() + - array.set_insert() + - array.set_union() + - array.set_intersection() + - array.set_difference() + - tbl.nth() + - tbl.for_each() + - tbl.get() + - r.expr([]).sample() + - tbl.offsets_of() + - ot: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + cd: + - r.db_create(1,2) + - r.db_drop(1,2) + - r.db(1,2) + - r.floor(1, 2) + - r.ceil(1, 2) + - r.round(1, 2) + + - cd: tbl.filter() + ot: + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + cd: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + + - cd: r.error(1, 2) + ot: err("ReqlCompileError", "Expected between 0 and 1 arguments but found 2.", []) + + - cd: db.table_drop() + ot: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + + + - cd: db.table_create() + ot: + cd: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + + - cd: r.js(1,2) + ot: + cd: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", []) + + - ot: err("ReqlCompileError", "Expected 2 arguments but found 3.", []) + cd: + - tbl.concat_map(1,2) + - tbl.skip(1,2) + - tbl.limit(1,2) + - array.append(1,2) + - array.prepend(1,2) + - array.difference([], []) + - array.set_insert(1,2) + - array.set_union([1],[2]) + - array.set_intersection([1],[2]) + - array.set_difference([1],[2]) + - tbl.nth(1,2) + - tbl.for_each(1,2) + - tbl.get(1,2) + - r.expr([]).sample(1,2) + - tbl.offsets_of(1,2) + + - cd: tbl.filter(1,2,3) + ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 4.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 3.", []) + + - cd: db.table_drop(1,2) + ot: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + + - cd: r.expr([]).delete_at() + ot: err("ReqlCompileError", "Expected between 2 and 3 arguments but found 1.", []) + + - cd: db.table_create(1,2) + ot: + cd: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", []) + + - cd: tbl.count(1,2) + ot: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + + - ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + cd: + - tbl.update() + - tbl.replace() + - tbl.insert() + + - cd: db.table() + ot: + cd: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", []) + + - cd: tbl.reduce() + ot: err("ReqlCompileError", "Expected 2 arguments but found 1.", []) + + - cd: tbl.eq_join() + ot: + cd: err("ReqlCompileError", "Expected 3 arguments but found 1.", []) + js: err("ReqlCompileError", "Expected 2 arguments (not including options) but found 0.", []) + + - ot: err("ReqlCompileError", "Expected 3 arguments but found 1.", []) + cd: + - tbl.inner_join() + - tbl.outer_join() + - r.expr([]).insert_at() + - r.expr([]).splice_at() + - r.expr([]).change_at() + + - cd: tbl.eq_join(1) + ot: + cd: err("ReqlCompileError", "Expected 3 arguments but found 2.", []) + js: err("ReqlCompileError", "Expected 2 arguments (not including options) but found 1.", []) + + - ot: err("ReqlCompileError", "Expected 3 arguments but found 2.", []) + cd: + - tbl.inner_join(1) + - tbl.outer_join(1) + - r.expr([]).insert_at(1) + - r.expr([]).splice_at(1) + - r.expr([]).change_at(1) + + - cd: tbl.eq_join(1,2,3,4) + ot: + cd: err("ReqlCompileError", "Expected 3 arguments but found 5.", []) + js: err("ReqlCompileError", "Expected 2 arguments (not including options) but found 4.", []) + + - ot: err("ReqlCompileError", "Expected 3 arguments but found 4.", []) + cd: + - tbl.inner_join(1,2,3) + - tbl.outer_join(1,2,3) + - r.expr([]).insert_at(1, 2, 3) + - r.expr([]).splice_at(1, 2, 3) + - r.expr([]).change_at(1, 2, 3) + + - cd: tbl.map() + ot: + cd: err('ReqlCompileError', "Expected 2 or more arguments but found 1.", []) + js: err('ReqlCompileError', "Expected 1 or more arguments but found 0.", []) + + - cd: r.branch(1,2) + ot: err("ReqlCompileError", "Expected 3 or more arguments but found 2.", []) + - cd: r.branch(1,2,3,4) + ot: err("ReqlQueryLogicError", "Cannot call `branch` term with an even number of arguments.", []) + + - cd: r.expr({})[1,2] + js: r.expr({})(1,2) + ot: + js: err('ReqlCompileError', "Expected 1 argument but found 2.", []) + py: err('ReqlQueryLogicError', 'Expected NUMBER or STRING as second argument to `bracket` but found ARRAY.') + rb: err('ArgumentError', 'wrong number of arguments (2 for 1)') + + - cd: tbl.insert([{'id':0},{'id':1},{'id':2},{'id':3},{'id':4},{'id':5},{'id':6},{'id':7},{'id':8},{'id':9}]).get_field('inserted') + ot: 10 + + - cd: tbl.get_all(0, 1, 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([]), 0, 1, 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0]), 1, 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0, 1]), 2).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0, 1, 2])).get_field('id') + ot: bag([0, 1, 2]) + + - cd: tbl.get_all(r.args([0]), 1, r.args([2])).get_field('id') + ot: bag([0, 1, 2]) + + # Make sure partial-evaluation still works + + - cd: r.branch(true, 1, r.error("a")) + ot: 1 + + - cd: r.branch(r.args([true, 1]), r.error("a")) + ot: 1 + + - cd: r.expr(true).branch(1, 2) + ot: 1 + + - cd: r.branch(r.args([true, 1, r.error("a")])) + ot: err("ReqlUserError", "a", []) + + # Make sure our grouped data hack still works + + - rb: tbl.group{|row| row['id'] % 2}.count({'id':0}).ungroup() + py: tbl.group(lambda row:row['id'].mod(2)).count({'id':0}).ungroup() + js: tbl.group(r.row('id').mod(2)).count({'id':0}).ungroup() + ot: ([{'group':0, 'reduction':1}]) + + - rb: tbl.group{|row| row['id'] % 2}.count(r.args([{'id':0}])).ungroup() + py: tbl.group(r.row['id'].mod(2)).count(r.args([{'id':0}])).ungroup() + js: tbl.group(r.row('id').mod(2)).count(r.args([{'id':0}])).ungroup() + ot: ([{'group':0, 'reduction':1}]) + + # Make sure `r.literal` still works + + - cd: r.expr({'a':{'b':1}}).merge(r.args([{'a':r.literal({'c':1})}])) + ot: ({'a':{'c':1}}) + + - cd: r.http("httpbin.org/get","bad_param") + ot: + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", []) + rb: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + py: err_regex('TypeError', ".*takes exactly 1 argument \(2 given\)", []) + py3.0: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.1: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.2: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3: err_regex('TypeError', ".*takes 1 positional argument but 2 were given", []) + + - cd: r.binary("1", "2") + ot: + py: err_regex('TypeError', ".*takes exactly 1 argument \(2 given\)", []) + js: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + rb: err("ReqlCompileError", "Expected 1 argument but found 2.", []) + py3.0: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.1: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3.2: err_regex('TypeError', ".*takes exactly 1 positional argument \(2 given\)", []) + py3: err_regex('TypeError', ".*takes 1 positional argument but 2 were given", []) + - cd: r.binary() + ot: + py: err_regex('TypeError', ".*takes exactly 1 argument \(0 given\)", []) + js: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + rb: err("ReqlCompileError", "Expected 1 argument but found 0.", []) + py3.0: err_regex('TypeError', ".*takes exactly 1 positional argument \(0 given\)", []) + py3.1: err_regex('TypeError', ".*takes exactly 1 positional argument \(0 given\)", []) + py3.2: err_regex('TypeError', ".*takes exactly 1 argument \(0 given\)", []) + py3: err_regex('TypeError', ".* missing 1 required positional argument.*", []) + + # TODO: Math and logic + # TODO: Upper bound on optional arguments + # TODO: between, merge, slice + + - cd: tbl.index_rename('idx') + ot: + cd: err('ReqlCompileError','Expected 3 arguments but found 2.',[]) + js: err('ReqlCompileError','Expected 2 arguments (not including options) but found 1.',[]) + + - cd: tbl.index_rename('idx','idx2','idx3') + ot: + cd: err('ReqlCompileError','Expected 3 arguments but found 4.',[]) + js: err('ReqlCompileError','Expected 2 arguments (not including options) but found 3.',[]) + + - cd: + - r.now('foo') + - r.now(r.args([1,2,3])) + ot: err('ReqlCompileError','NOW does not accept any args.') diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml new file mode 100644 index 00000000..e66b6847 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/edge.yaml @@ -0,0 +1,142 @@ +desc: Test edge cases of changefeed operations +table_variable_name: tbl +tests: + + - def: common_prefix = r.expr([0,1,2,3,4,5,6,7,8]) + + - js: tbl.indexCreate('sindex', function (row) { return common_prefix.append(row('value')); }) + py: tbl.index_create('sindex', lambda row:common_prefix.append(row['value'])) + rb: tbl.index_create('sindex'){ |row| common_prefix.append(row['value']) } + ot: ({'created':1}) + - cd: tbl.index_wait('sindex') + + # create target values + - cd: pre = r.range(7).coerce_to('array').add(r.range(10,70).coerce_to('array')).append(100).map(r.row.coerce_to('string')) + rb: pre = r.range(7).coerce_to('array').add(r.range(10,70).coerce_to('array')).append(100).map{ |row| row.coerce_to('string') } + - cd: mid = r.range(2,9).coerce_to('array').add(r.range(20,90).coerce_to('array')).map(r.row.coerce_to('string')) + rb: mid = r.range(2,9).coerce_to('array').add(r.range(20,90).coerce_to('array')).map{ |row| row.coerce_to('string') } + - cd: post = r.range(3,10).coerce_to('array').add(r.range(30,100).coerce_to('array')).map(r.row.coerce_to('string')) + rb: post = r.range(3,10).coerce_to('array').add(r.range(30,100).coerce_to('array')).map{ |row| row.coerce_to('string') } + + - cd: erroredres = r.range(2).coerce_to('array').add(r.range(10, 20).coerce_to('array')).append(100).map(r.row.coerce_to('string')) + rb: erroredres = r.range(2).coerce_to('array').add(r.range(10, 20).coerce_to('array')).append(100).map{ |val| val.coerce_to('string') } + + # Start overlapping changefeeds + - js: pre_changes = tbl.between(r.minval, commonPrefix.append('7'), {index:'sindex'}).changes({squash:false}).limit(pre.length)('new_val')('value') + py: pre_changes = tbl.between(r.minval, common_prefix.append('7'), index='sindex').changes(squash=False).limit(len(pre))['new_val']['value'] + rb: pre_changes = tbl.between(r.minval, common_prefix.append('7'), index:'sindex').changes(squash:false).limit(pre.length)['new_val']['value'] + - js: mid_changes = tbl.between(commonPrefix.append('2'), common_prefix.append('9'), {index:'sindex'}).changes({squash:false}).limit(post.length)('new_val')('value') + py: mid_changes = tbl.between(common_prefix.append('2'), common_prefix.append('9'), index='sindex').changes(squash=False).limit(len(post))['new_val']['value'] + rb: mid_changes = tbl.between(common_prefix.append('2'), common_prefix.append('9'), index:'sindex').changes(squash:false).limit(post.length)['new_val']['value'] + - js: post_changes = tbl.between(commonPrefix.append('3'), r.maxval, {index:'sindex'}).changes({squash:false}).limit(mid.length)('new_val')('value') + py: post_changes = tbl.between(common_prefix.append('3'), r.maxval, index='sindex').changes(squash=False).limit(len(mid))['new_val']['value'] + rb: post_changes = tbl.between(common_prefix.append('3'), r.maxval, index:'sindex').changes(squash:false).limit(mid.length)['new_val']['value'] + + # Start changefeeds with non-existence errors + + - js: premap_changes1 = tbl.map(r.branch(r.row('value').lt('2'), r.row, r.row("dummy"))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: premap_changes1 = tbl.map(r.branch(r.row['value'].lt('2'), r.row, r.row["dummy"])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: premap_changes1 = tbl.map{ |row| r.branch(row['value'].lt('2'), row, row["dummy"]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postmap_changes1 = tbl.changes({squash:false}).map(r.branch(r.row('new_val')('value').lt('2'), r.row, r.row("dummy"))).limit(erroredres.length)('new_val')('value') + py: postmap_changes1 = tbl.changes(squash=False).map(r.branch(r.row['new_val']['value'].lt('2'), r.row, r.row["dummy"])).limit(len(erroredres))['new_val']['value'] + rb: postmap_changes1 = tbl.changes(squash:false).map{ |row| r.branch(row['new_val']['value'].lt('2'), row, row["dummy"]) }.limit(erroredres.length)['new_val']['value'] + + - js: prefilter_changes1 = tbl.filter(r.branch(r.row('value').lt('2'), true, r.row("dummy"))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: prefilter_changes1 = tbl.filter(r.branch(r.row['value'].lt('2'), True, r.row["dummy"])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: prefilter_changes1 = tbl.filter{ |row| r.branch(row['value'].lt('2'), true, row["dummy"]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postfilter_changes1 = tbl.changes({squash:false}).filter(r.branch(r.row('new'+'_'+'val')('value').lt('2'), true, r.row("dummy"))).limit(erroredres.length)('new_val')('value') + py: postfilter_changes1 = tbl.changes(squash=False).filter(r.branch(r.row['new_val']['value'].lt('2'), True, r.row["dummy"])).limit(len(erroredres))['new_val']['value'] + rb: postfilter_changes1 = tbl.changes(squash:false).filter{ |row| r.branch(row['new_val']['value'].lt('2'), true, row["dummy"]) }.limit(erroredres.length)['new_val']['value'] + + # Start changefeeds with runtime errors + + - js: premap_changes2 = tbl.map(r.branch(r.row('value').lt('2'), r.row, r.expr([]).nth(1))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: premap_changes2 = tbl.map(r.branch(r.row['value'].lt('2'), r.row, r.expr([])[1])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: premap_changes2 = tbl.map{ |row| r.branch(row['value'].lt('2'), row, r.expr([])[1]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postmap_changes2 = tbl.changes({squash:false}).map(r.branch(r.row('new'+'_'+'val')('value').lt('2'), r.row, r.expr([]).nth(1))).limit(erroredres.length)('new_val')('value') + py: postmap_changes2 = tbl.changes(squash=False).map(r.branch(r.row['new_val']['value'].lt('2'), r.row, r.expr([])[1])).limit(len(erroredres))['new_val']['value'] + rb: postmap_changes2 = tbl.changes(squash:false).map{ |row| r.branch(row['new_val']['value'].lt('2'), row, r.expr([])[1]) }.limit(erroredres.length)['new_val']['value'] + + - js: prefilter_changes2 = tbl.filter(r.branch(r.row('value').lt('2'), true, r.expr([]).nth(1))).changes({squash:false}).limit(erroredres.length)('new_val')('value') + py: prefilter_changes2 = tbl.filter(r.branch(r.row['value'].lt('2'), True, r.expr([])[1])).changes(squash=False).limit(len(erroredres))['new_val']['value'] + rb: prefilter_changes2 = tbl.filter{ |row| r.branch(row['value'].lt('2'), true, r.expr([])[1]) }.changes(squash:false).limit(erroredres.length)['new_val']['value'] + + - js: postfilter_changes2 = tbl.changes({squash:false}).filter(r.branch(r.row('new'+'_'+'val')('value').lt('2'), true, r.expr([]).nth(1))).limit(erroredres.length)('new_val')('value') + py: postfilter_changes2 = tbl.changes(squash=False).filter(r.branch(r.row['new_val']['value'].lt('2'), True, r.expr([])[1])).limit(len(erroredres))['new_val']['value'] + rb: postfilter_changes2 = tbl.changes(squash:false).filter{ |row| r.branch(row['new_val']['value'].lt('2'), true, r.expr([])[1]) }.limit(erroredres.length)['new_val']['value'] + + # Start non-deterministic changefeeds - very small chance of these hanging due to not enough results + - def: + py: nondetermmap = r.branch(r.random().gt(0.5), r.row, r.error("dummy")) + js: nondetermmap = function (row) { return r.branch(r.random().gt(0.5), row, r.error("dummy")); } + rb: nondetermmap = Proc.new { |row| r.branch(r.random().gt(0.5), row, r.error("dummy")) } + - def: + py: nondetermfilter = lambda row:r.random().gt(0.5) + js: nondetermfilter = function (row) { return r.random().gt(0.5); } + rb: nondetermfilter = Proc.new { |row| r.random().gt(0.5) } + + - rb: tbl.map(nondetermmap).changes(squash:false) + js: tbl.map(nondetermmap).changes({squash:false}) + py: tbl.map(nondetermmap).changes(squash=False) + ot: err('ReqlQueryLogicError', 'Cannot call `changes` after a non-deterministic function.') + + - rb: postmap_changes3 = tbl.changes(squash:false).map(nondetermmap).limit(100) + js: postmap_changes3 = tbl.changes({squash:false}).map(nondetermmap).limit(100) + py: postmap_changes3 = tbl.changes(squash=False).map(nondetermmap).limit(100) + + - rb: tbl.filter(nondetermfilter).changes(squash:false) + js: tbl.filter(nondetermfilter).changes({squash:false}) + py: tbl.filter(nondetermfilter).changes(squash=False) + ot: err('ReqlQueryLogicError', 'Cannot call `changes` after a non-deterministic function.') + + - rb: postfilter_changes3 = tbl.changes(squash:false).filter(nondetermfilter).limit(4) + js: postfilter_changes3 = tbl.changes({squash:false}).filter(nondetermfilter).limit(4) + py: postfilter_changes3 = tbl.changes(squash=False).filter(nondetermfilter).limit(4) + + # Insert several rows that will and will not be returned + - cd: tbl.insert(r.range(101).map({'id':r.uuid().coerce_to('binary').slice(0,r.random(4,24)).coerce_to('string'),'value':r.row.coerce_to('string')})) + rb: tbl.insert(r.range(101).map{ |row| {'id'=>r.uuid().coerce_to('binary').slice(0,r.random(4,24)).coerce_to('string'),'value'=>row.coerce_to('string')}}) + ot: ({'skipped':0,'deleted':0,'unchanged':0,'errors':0,'replaced':0,'inserted':101}) + + # Check that our limited watchers have been satified + - cd: pre_changes + ot: bag(pre) + + - cd: mid_changes + ot: bag(mid) + + - cd: post_changes + ot: bag(post) + + - cd: premap_changes1 + ot: bag(erroredres) + + - cd: premap_changes2 + ot: bag(erroredres) + + - cd: postmap_changes1 + ot: err('ReqlNonExistenceError', "No attribute `dummy` in object:") + + - cd: postmap_changes2 + ot: err('ReqlNonExistenceError', "Index out of bounds:" + " 1") + + - cd: postmap_changes3 + ot: err('ReqlUserError', "dummy") + + - cd: prefilter_changes1 + ot: bag(erroredres) + + - cd: prefilter_changes2 + ot: bag(erroredres) + + - cd: postfilter_changes1 + ot: bag(erroredres) + + - cd: postfilter_changes2 + ot: bag(erroredres) + + - ot: arrlen(postfilter_changes3) + ot: 4 diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml new file mode 100644 index 00000000..7755ff96 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/geo.rb.yaml @@ -0,0 +1,27 @@ +desc: Geo indexed changefeed operations +table_variable_name: tbl +tests: + - rb: tbl.index_create('L', {geo: true}) + ot: partial({'created': 1}) + + - rb: tbl.index_wait().count + ot: 1 + + - def: obj11 = {id: "11", L: r.point(1,1)} + - def: obj12 = {id: "12", L: r.point(1,2)} + - def: obj21 = {id: "21", L: r.point(2,1)} + - def: obj22 = {id: "22", L: r.point(2,2)} + + # A distance of 130,000 meters from 1,1 is enough to cover 1,2 and 2,1 (~110km + # distance) but not 2,2 (~150km distance.) + # + # This is useful because the S2LatLngRect bounding box passed to the shards contains + # 2,2 yet it should not be returned in the changefeed results. + - rb: feed = tbl.get_intersecting(r.circle(r.point(1,1), 130000), {index: "L"}).get_field("id").changes(include_initial: true) + + - rb: tbl.insert([obj11, obj12, obj21, obj22]) + ot: partial({'errors': 0, 'inserted': 4}) + + - rb: fetch(feed, 3) + ot: bag([{"new_val" => "11", "old_val" => nil}, {"new_val" => "12", "old_val" => nil}, {"new_val" => "21", "old_val" => nil}]) + diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml new file mode 100644 index 00000000..f7110ab1 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/idxcopy.yaml @@ -0,0 +1,38 @@ +desc: Test duplicate indexes with squashing +table_variable_name: tbl +tests: + - cd: tbl.index_create('a') + ot: partial({'created':1}) + - cd: tbl.index_wait('a') + + - py: feed = tbl.order_by(index='a').limit(10).changes(squash=2) + rb: feed = tbl.orderby(index:'a').limit(10).changes(squash:2).limit(9) + js: feed = tbl.orderBy({index:'a'}).limit(10).changes({squash:2}).limit(9) + runopts: + # limit the number of pre-fetched rows + max_batch_rows: 1 + + - py: tbl.insert(r.range(0, 12).map({'id':r.row, 'a':5})) + rb: tbl.insert(r.range(0, 12).map{|row| {'id':row, 'a':5}}) + js: tbl.insert(r.range(0, 12).map(function(row){ return {'id':row, 'a':5}; })) + ot: partial({'inserted':12, 'errors':0}) + + - py: tbl.get_all(1, 8, 9, index='id').delete() + rb: tbl.get_all(1, 8, 9, index:'id').delete() + js: tbl.get_all(1, 8, 9, {index:'id'}).delete() + ot: partial({'deleted':3, 'errors':0}) + + # should be replaced with a noreplyWait + - cd: wait(2) + + - cd: fetch(feed) + ot: bag([ + {"new_val":{"a":5, "id":0}, "old_val":nil}, + {"new_val":{"a":5, "id":2}, "old_val":nil}, + {"new_val":{"a":5, "id":3}, "old_val":nil}, + {"new_val":{"a":5, "id":4}, "old_val":nil}, + {"new_val":{"a":5, "id":5}, "old_val":nil}, + {"new_val":{"a":5, "id":6}, "old_val":nil}, + {"new_val":{"a":5, "id":7}, "old_val":nil}, + {"new_val":{"a":5, "id":10}, "old_val":nil}, + {"new_val":{"a":5, "id":11}, "old_val":nil}]) diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml new file mode 100644 index 00000000..68dadb08 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/include_states.yaml @@ -0,0 +1,58 @@ +desc: Test `include_states` +table_variable_name: tbl +tests: + - py: tbl.changes(squash=true, include_states=true).limit(1) + rb: tbl.changes(squash:true, include_states:true).limit(1) + js: tbl.changes({squash:true, includeStates:true}).limit(1) + ot: [{'state':'ready'}] + + - py: tbl.get(0).changes(squash=true, include_states=true, include_initial=true).limit(3) + rb: tbl.get(0).changes(squash:true, include_states:true, include_initial:true).limit(3) + js: tbl.get(0).changes({squash:true, includeStates:true, includeInitial:true}).limit(3) + ot: [{'state':'initializing'}, {'new_val':null}, {'state':'ready'}] + + - py: tbl.order_by(index='id').limit(10).changes(squash=true, include_states=true, include_initial=true).limit(2) + rb: tbl.order_by(index:'id').limit(10).changes(squash:true, include_states:true, include_initial:true).limit(2) + js: tbl.orderBy({index:'id'}).limit(10).changes({squash:true, includeStates:true, includeInitial:true}).limit(2) + ot: [{'state':'initializing'}, {'state':'ready'}] + + - cd: tbl.insert({'id':1}) + + - py: tbl.order_by(index='id').limit(10).changes(squash=true, include_states=true, include_initial=true).limit(3) + rb: tbl.order_by(index:'id').limit(10).changes(squash:true, include_states:true, include_initial:true).limit(3) + js: tbl.orderBy({index:'id'}).limit(10).changes({squash:true, includeStates:true, includeInitial:true}).limit(3) + ot: [{'state':'initializing'}, {'new_val':{'id':1}}, {'state':'ready'}] + + - py: tblchanges = tbl.changes(squash=true, include_states=true) + rb: tblchanges = tbl.changes(squash:true, include_states:true) + js: tblchanges = tbl.changes({squash:true, includeStates:true}) + + - cd: tbl.insert({'id':2}) + + - cd: fetch(tblchanges, 2) + ot: [{'state':'ready'},{'new_val':{'id':2},'old_val':null}] + + - py: getchanges = tbl.get(2).changes(include_states=true, include_initial=true) + rb: getchanges = tbl.get(2).changes(include_states:true, include_initial:true) + js: getchanges = tbl.get(2).changes({includeStates:true, includeInitial:true}) + + - cd: tbl.get(2).update({'a':1}) + + - cd: fetch(getchanges, 4) + ot: [{'state':'initializing'}, {'new_val':{'id':2}}, {'state':'ready'}, {'old_val':{'id':2},'new_val':{'id':2,'a':1}}] + + - py: limitchanges = tbl.order_by(index='id').limit(10).changes(include_states=true, include_initial=true) + rb: limitchanges = tbl.order_by(index:'id').limit(10).changes(include_states:true, include_initial:true) + js: limitchanges = tbl.orderBy({index:'id'}).limit(10).changes({includeStates:true, includeInitial:true}) + + - py: limitchangesdesc = tbl.order_by(index=r.desc('id')).limit(10).changes(include_states=true, include_initial=true) + rb: limitchangesdesc = tbl.order_by(index:r.desc('id')).limit(10).changes(include_states:true, include_initial:true) + js: limitchangesdesc = tbl.orderBy({index:r.desc('id')}).limit(10).changes({includeStates:true, includeInitial:true}) + + - cd: tbl.insert({'id':3}) + + - cd: fetch(limitchanges, 5) + ot: [{'state':'initializing'}, {'new_val':{'id':1}}, {'new_val':{'a':1, 'id':2}}, {'state':'ready'}, {'old_val':null, 'new_val':{'id':3}}] + + - cd: fetch(limitchangesdesc, 5) + ot: [{'state':'initializing'}, {'new_val':{'a':1, 'id':2}}, {'new_val':{'id':1}}, {'state':'ready'}, {'old_val':null, 'new_val':{'id':3}}] diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/point.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/point.yaml new file mode 100644 index 00000000..a444a73d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/point.yaml @@ -0,0 +1,147 @@ +desc: Test point changebasics +table_variable_name: tbl +tests: + + # -- basic + + # start a feed + + - cd: basic = tbl.get(1).changes({include_initial:true}) + py: basic = tbl.get(1).changes(include_initial=True) + + # - inital return + + - cd: fetch(basic, 1) + ot: [{'new_val':null}] + + # - inserts + + - cd: tbl.insert({'id':1}) + ot: partial({'errors':0, 'inserted':1}) + + - cd: fetch(basic, 1) + ot: [{'old_val':null, 'new_val':{'id':1}}] + + # - updates + + - cd: tbl.get(1).update({'update':1}) + ot: partial({'errors':0, 'replaced':1}) + + - cd: fetch(basic, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1,'update':1}}] + + # - deletions + + - cd: tbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + + - cd: fetch(basic, 1) + ot: [{'old_val':{'id':1,'update':1}, 'new_val':null}] + + # - closing + + - cd: basic.close() + rb: def pass; end + # the ruby test driver currently has to mangle cursors, so we can't close them properly + + # -- filter + + - py: filter = tbl.get(1).changes(squash=false,include_initial=True).filter(r.row['new_val']['update'].gt(2))['new_val']['update'] + rb: filter = tbl.get(1).changes(squash:false,include_initial:true).filter{|row| row['new_val']['update'].gt(2)}['new_val']['update'] + js: filter = tbl.get(1).changes({squash:false,include_initial:true}).filter(r.row('new_val')('update').gt(2))('new_val')('update') + + - cd: tbl.insert({'id':1, 'update':1}) + - cd: tbl.get(1).update({'update':4}) + - cd: tbl.get(1).update({'update':1}) + - cd: tbl.get(1).update({'update':7}) + + - cd: fetch(filter, 2) + ot: [4,7] + + # -- pluck on values + + - py: pluck = tbl.get(3).changes(squash=false,include_initial=True).pluck({'new_val':['red', 'blue']})['new_val'] + rb: pluck = tbl.get(3).changes(squash:false,include_initial:true).pluck({'new_val':['red', 'blue']})['new_val'] + js: pluck = tbl.get(3).changes({squash:false,include_initial:true}).pluck({'new_val':['red', 'blue']})('new_val') + + - cd: tbl.insert({'id':3, 'red':1, 'green':1}) + ot: partial({'errors':0, 'inserted':1}) + - cd: tbl.get(3).update({'blue':2, 'green':3}) + ot: partial({'errors':0, 'replaced':1}) + - cd: tbl.get(3).update({'green':4}) + ot: partial({'errors':0, 'replaced':1}) + - cd: tbl.get(3).update({'blue':4}) + ot: partial({'errors':0, 'replaced':1}) + + - cd: fetch(pluck, 4) + ot: [{'red': 1}, {'blue': 2, 'red': 1}, {'blue': 2, 'red': 1}, {'blue': 4, 'red': 1}] + + # -- virtual tables + + # - rethinkdb._debug_scratch + + - def: dtbl = r.db('rethinkdb').table('_debug_scratch') + + - cd: debug = dtbl.get(1).changes({include_initial:true}) + py: debug = dtbl.get(1).changes(include_initial=True) + + - cd: fetch(debug, 1) + ot: [{'new_val':null}] + + - cd: dtbl.insert({'id':1}) + ot: partial({'errors':0, 'inserted':1}) + - cd: fetch(debug, 1) + ot: [{'old_val':null, 'new_val':{'id':1}}] + + - cd: dtbl.get(1).update({'update':1}) + ot: partial({'errors':0, 'replaced':1}) + - cd: fetch(debug, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1,'update':1}}] + + - cd: dtbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + - cd: fetch(debug, 1) + ot: [{'old_val':{'id':1,'update':1}, 'new_val':null}] + + - cd: dtbl.insert({'id':5, 'red':1, 'green':1}) + ot: {'skipped':0, 'deleted':0, 'unchanged':0, 'errors':0, 'replaced':0, 'inserted':1} + - py: dtblPluck = dtbl.get(5).changes(include_initial=True).pluck({'new_val':['red', 'blue']})['new_val'] + rb: dtblPluck = dtbl.get(5).changes(include_initial:true).pluck({'new_val':['red', 'blue']})['new_val'] + js: dtblPluck = dtbl.get(5).changes({include_initial:true}).pluck({'new_val':['red', 'blue']})('new_val') + + # disabled because inital value is not being reported correctly, so goes missing. see #3723 + - cd: fetch(dtblPluck, 1) + ot: [{'red':1}] + + - cd: dtbl.get(5).update({'blue':2, 'green':3}) + ot: partial({'errors':0, 'replaced':1}) + + - cd: fetch(dtblPluck, 1) + ot: [{'blue':2, 'red':1}] + + # - rethinkdb.table_status bad optargs + + # disabled, re-enable once #3725 is done + # - py: r.db('rethinkdb').table('table_status').changes(squash=False) + # rb: r.db('rethinkdb').table('table_status').changes(squash:False) + # js: r.db('rethinkdb').table('table_status').changes({squash:False}) + # ot: err('ReqlRuntimeError', 'replace with error message decided in \#3725') + + # - rethinkdb.table_status + + - cd: tableId = tbl.info()['id'] + js: tableId = tbl.info()('id') + + - cd: rtblPluck = r.db('rethinkdb').table('table_status').get(tableId).changes({include_initial:true}) + py: rtblPluck = r.db('rethinkdb').table('table_status').get(tableId).changes(include_initial=True) + - cd: fetch(rtblPluck, 1) + ot: partial([{'new_val':partial({'db':'test'})}]) + + - py: tbl.reconfigure(shards=3, replicas=1) + rb: tbl.reconfigure(shards:3, replicas:1) + js: tbl.reconfigure({shards:3, replicas:1}) + - py: fetch(rtblPluck, 1, 2) + js: fetch(rtblPluck, 1, 2) + rb: fetch(rtblPluck, 1) + ot: partial([{'old_val':partial({'db':'test'}), 'new_val':partial({'db':'test'})}]) + diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml new file mode 100644 index 00000000..85ef969d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/sindex.yaml @@ -0,0 +1,50 @@ +desc: Test basic changefeed operations +table_variable_name: tbl +tests: + + # Fill in some data + - rb: tbl.index_create('a') + ot: partial({'created':1}) + + - rb: tbl.index_wait().count + ot: 1 + + - rb: tbl.insert([{id:1, a:8}, {id:2, a:7}]) + ot: partial({'errors':0, 'inserted':2}) + + - rb: idmin = tbl.min(index:'id').changes(squash:false, include_initial:true).limit(2) + - rb: idmax = tbl.max(index:'id').changes(squash:false, include_initial:true).limit(2) + - rb: amin = tbl.min(index:'a').changes(squash:false, include_initial:true).limit(2) + - rb: amax = tbl.max(index:'a').changes(squash:false, include_initial:true).limit(2) + + - rb: idmin2 = tbl.min(index:'id').changes(squash:true, include_initial:true).limit(2) + - rb: idmax2 = tbl.max(index:'id').changes(squash:true, include_initial:true).limit(2) + - rb: amin2 = tbl.min(index:'a').changes(squash:true, include_initial:true).limit(2) + - rb: amax2 = tbl.max(index:'a').changes(squash:true, include_initial:true).limit(2) + + - rb: tbl.insert([{id:0, a:9}, {id:3, a:6}]) + ot: partial({'errors':0, 'inserted':2}) + + - rb: idmin.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) + + - rb: idmax.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amin.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amax.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) + + - rb: idmin2.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) + + - rb: idmax2.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amin2.to_a + ot: ([{"new_val"=>{"a"=>7, "id"=>2}}, {"new_val"=>{"a"=>6, "id"=>3}, "old_val"=>{"a"=>7, "id"=>2}}]) + + - rb: amax2.to_a + ot: ([{"new_val"=>{"a"=>8, "id"=>1}}, {"new_val"=>{"a"=>9, "id"=>0}, "old_val"=>{"a"=>8, "id"=>1}}]) diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml new file mode 100644 index 00000000..fa4ad7f0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/squash.yaml @@ -0,0 +1,62 @@ +desc: Test changefeed squashing +table_variable_name: tbl +tests: + + # Check type + + - py: tbl.changes(squash=true).type_of() + rb: tbl.changes(squash:true).type_of() + js: tbl.changes({squash:true}).typeOf() + ot: ("STREAM") + + # comparison changes + + - cd: normal_changes = tbl.changes().limit(2) + + - py: false_squash_changes = tbl.changes(squash=False).limit(2) + js: false_squash_changes = tbl.changes({squash:false}).limit(2) + rb: false_squash_changes = tbl.changes(squash:false).limit(2) + + - py: long_squash_changes = tbl.changes(squash=0.5).limit(1) + js: long_squash_changes = tbl.changes({squash:0.5}).limit(1) + rb: long_squash_changes = tbl.changes(squash:0.5).limit(1) + + - py: squash_changes = tbl.changes(squash=true).limit(1) + js: squash_changes = tbl.changes({squash:true}).limit(1) + rb: squash_changes = tbl.changes(squash:true).limit(1) + + - cd: tbl.insert({'id':100})['inserted'] + js: tbl.insert({'id':100})('inserted') + ot: 1 + + - cd: tbl.get(100).update({'a':1})['replaced'] + js: tbl.get(100).update({'a':1})('replaced') + ot: 1 + + - cd: normal_changes + ot: ([{'new_val':{'id':100}, 'old_val':null}, + {'new_val':{'a':1, 'id':100}, 'old_val':{'id':100}}]) + + - cd: false_squash_changes + ot: ([{'new_val':{'id':100}, 'old_val':null}, + {'new_val':{'a':1, 'id':100}, 'old_val':{'id':100}}]) + + - cd: long_squash_changes + ot: ([{'new_val':{'a':1, 'id':100}, 'old_val':null}]) + + - cd: squash_changes + ot: + js: ([{'new_val':{'a':1, 'id':100}, 'old_val':null}]) + cd: ([{'new_val':{'id':100}, 'old_val':null}]) + + # Bad squash values + + - py: tbl.changes(squash=null) + rb: tbl.changes(squash:null) + js: tbl.changes({squash:null}) + ot: err('ReqlQueryLogicError', 'Expected BOOL or NUMBER but found NULL.') + + - py: tbl.changes(squash=-10) + rb: tbl.changes(squash:-10) + js: tbl.changes({squash:-10}) + ot: err('ReqlQueryLogicError', 'Expected BOOL or a positive NUMBER but found a negative NUMBER.') diff --git a/ext/librethinkdbxx/test/upstream/changefeeds/table.yaml b/ext/librethinkdbxx/test/upstream/changefeeds/table.yaml new file mode 100644 index 00000000..c089abc9 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/changefeeds/table.yaml @@ -0,0 +1,101 @@ +desc: Test changefeeds on a table +table_variable_name: tbl +tests: + + # ==== regular tables + + # - start feeds + + - cd: all = tbl.changes() + + # - note: no initial values from table changefeeds + + # - inserts + + - cd: tbl.insert([{'id':1}, {'id':2}]) + ot: partial({'errors':0, 'inserted':2}) + - cd: fetch(all, 2) + ot: bag([{'old_val':null, 'new_val':{'id':1}}, {'old_val':null, 'new_val':{'id':2}}]) + + # - updates + + - cd: tbl.get(1).update({'version':1}) + ot: partial({'errors':0, 'replaced':1}) + - cd: fetch(all, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1, 'version':1}}] + + # - deletions + + - cd: tbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + - cd: fetch(all, 1) + ot: [{'old_val':{'id':1, 'version':1}, 'new_val':null}] + + # - pluck on values + + - cd: pluck = tbl.changes().pluck({'new_val':['version']}) + - cd: tbl.insert([{'id':5, 'version':5}]) + ot: partial({'errors':0, 'inserted':1}) + - cd: fetch(pluck, 1) + ot: [{'new_val':{'version':5}}] + + # - order by + + - cd: tbl.changes().order_by('id') + ot: err('ReqlQueryLogicError', "Cannot call a terminal (`reduce`, `count`, etc.) on an infinite stream (such as a changefeed).") +# +# ToDo: enable this when #4067 is done +# +# - js: orderedLimit = tbl.changes().limit(5).order_by(r.desc('id'))('new_val')('id') +# cd: orderedLimit = tbl.changes().limit(5).order_by(r.desc('id'))['new_val']['id'] +# - js: tbl.range(100, 105).map(function (row) { return {'id':row} }) +# py: tbl.range(100, 105).map({'id':r.row}) +# rb: tbl.range(100, 105).map{|row| {'id':row}} +# - cd: fetch(orderedLimit) +# ot: [104, 103, 102, 101, 100] + + # - changes overflow + + - cd: overflow = tbl.changes() + runopts: + changefeed_queue_size: 100 + # add enough entries to make sure we get the overflow error + - js: tbl.insert(r.range(200).map(function(x) { return({}); })) + py: tbl.insert(r.range(200).map(lambda x: {})) + rb: tbl.insert(r.range(200).map{|x| {}}) + - cd: fetch(overflow, 90) + ot: partial([{'error': regex('Changefeed cache over array size limit, skipped \d+ elements.')}]) + + # ==== virtual tables + + - def: vtbl = r.db('rethinkdb').table('_debug_scratch') + - cd: allVirtual = vtbl.changes() + + # - inserts + + - cd: vtbl.insert([{'id':1}, {'id':2}]) + ot: partial({'errors':0, 'inserted':2}) + - cd: fetch(allVirtual, 2) + ot: bag([{'old_val':null, 'new_val':{'id':1}}, {'old_val':null, 'new_val':{'id':2}}]) + + # - updates + + - cd: vtbl.get(1).update({'version':1}) + ot: partial({'errors':0, 'replaced':1}) + - cd: fetch(allVirtual, 1) + ot: [{'old_val':{'id':1}, 'new_val':{'id':1, 'version':1}}] + + # - deletions + + - cd: vtbl.get(1).delete() + ot: partial({'errors':0, 'deleted':1}) + - cd: fetch(allVirtual, 1) + ot: [{'old_val':{'id':1, 'version':1}, 'new_val':null}] + + # - pluck on values + + - cd: vpluck = vtbl.changes().pluck({'new_val':['version']}) + - cd: vtbl.insert([{'id':5, 'version':5}]) + ot: partial({'errors':0, 'inserted':1}) + - cd: fetch(vpluck, 1) + ot: [{'new_val':{'version':5}}] diff --git a/ext/librethinkdbxx/test/upstream/control.yaml b/ext/librethinkdbxx/test/upstream/control.yaml new file mode 100644 index 00000000..248d070a --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/control.yaml @@ -0,0 +1,297 @@ +desc: Tests RQL control flow structures +table_variable_name: tbl, tbl2 +tests: + + ## FunCall + + - py: r.expr(1).do(lambda v: v * 2) + js: r.expr(1).do(function(v) { return v.mul(2); }) + rb: r.expr(1).do{|v| v * 2 } + ot: 2 + + - py: r.expr([0, 1, 2]).do(lambda v: v.append(3)) + js: r([0, 1, 2]).do(function(v) { return v.append(3); }) + rb: r([0, 1, 2]).do{ |v| v.append(3) } + ot: [0, 1, 2, 3] + + - py: r.do(1, 2, lambda x, y: x + y) + js: r.do(1, 2, function(x, y) { return x.add(y); }) + rb: r.do(1, 2) {|x, y| x + y} + ot: 3 + + - py: r.do(lambda: 1) + js: r.do(function() { return 1; }) + rb: r.do{1} + ot: 1 + + # do error cases + - py: r.do(1, 2, lambda x: x) + js: r.do(1, 2, function(x) { return x; }) + rb: r.do(1, 2) {|x| x} + ot: err("ReqlQueryLogicError", 'Expected function with 2 arguments but found function with 1 argument.', [1]) + + - py: r.do(1, 2, 3, lambda x, y: x + y) + js: r.do(1, 2, 3, function(x, y) { return x.add(y); }) + rb: r.do(1, 2, 3) {|x, y| x + y} + ot: err("ReqlQueryLogicError", 'Expected function with 3 arguments but found function with 2 arguments.', [1]) + + - cd: r.do(1) + ot: 1 + + - js: r.do(1, function(x) {}) + ot: err("ReqlDriverCompileError", 'Anonymous function returned `undefined`. Did you forget a `return`?', [1]) + + - js: r.do(1, function(x) { return undefined; }) + ot: err("ReqlDriverCompileError", 'Anonymous function returned `undefined`. Did you forget a `return`?', [1]) + + - cd: r.do() + ot: + cd: err("ReqlCompileError", 'Expected 1 or more arguments but found 0.', [1]) + + # FunCall errors + + - py: r.expr('abc').do(lambda v: v.append(3)) + js: r('abc').do(function(v) { return v.append(3); }) + rb: r('abc').do{ |v| v.append(3) } + ot: err("ReqlQueryLogicError", "Expected type ARRAY but found STRING.", [1, 0]) + + - py: r.expr('abc').do(lambda v: v + 3) + js: r('abc').do(function(v) { return v.add(3); }) + rb: r('abc').do{ |v| v + 3 } + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [1, 1]) + + - py: r.expr('abc').do(lambda v: v + 'def') + 3 + js: r('abc').do(function(v) { return v.add('def'); }).add(3) + rb: r('abc').do{ |v| v + 'def' } + 3 + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [1]) + + - py: r.expr(0).do(lambda a,b: a + b) + js: r(0).do(function(a,b) { return a.add(b); }) + rb: r(0).do{ |a, b| a + b } + ot: err("ReqlQueryLogicError", 'Expected function with 1 argument but found function with 2 arguments.', [1]) + + - py: r.do(1, 2, lambda a: a) + js: r.do(1,2, function(a) { return a; }) + rb: r.do(1, 2) { |a| a } + ot: err("ReqlQueryLogicError", 'Expected function with 2 arguments but found function with 1 argument.', [1]) + + - cd: r.expr(5).do(r.row) + rb: r(5).do{ |row| row } + ot: 5 + + ## Branch + + - cd: r.branch(True, 1, 2) + ot: 1 + - cd: r.branch(False, 1, 2) + ot: 2 + - cd: r.branch(1, 'c', False) + ot: ("c") + - cd: r.branch(null, {}, []) + ot: ([]) + + - cd: r.branch(r.db('test'), 1, 2) + ot: err("ReqlQueryLogicError", "Expected type DATUM but found DATABASE:", []) + - cd: r.branch(tbl, 1, 2) + ot: err("ReqlQueryLogicError", "Expected type DATUM but found TABLE:", []) + - cd: r.branch(r.error("a"), 1, 2) + ot: err("ReqlUserError", "a", []) + + - cd: r.branch([], 1, 2) + ot: 1 + - cd: r.branch({}, 1, 2) + ot: 1 + - cd: r.branch("a", 1, 2) + ot: 1 + - cd: r.branch(1.2, 1, 2) + ot: 1 + + - cd: r.branch(True, 1, True, 2, 3) + ot: 1 + - cd: r.branch(True, 1, False, 2, 3) + ot: 1 + - cd: r.branch(False, 1, True, 2, 3) + ot: 2 + - cd: r.branch(False, 1, False, 2, 3) + ot: 3 + + - cd: r.branch(True, 1, True, 2) + ot: err("ReqlQueryLogicError", "Cannot call `branch` term with an even number of arguments.") + + # r.error() + - cd: r.error('Hello World') + ot: err("ReqlUserError", "Hello World", [0]) + + - cd: r.error(5) + # we might want to allow this eventually + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [0]) + + # r.filter + - cd: r.expr([1, 2, 3]).filter() + ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 1.", [0]) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 0.", [0]) + - cd: r.expr([1, 2, 3]).filter(1, 2) + ot: + cd: err("ReqlCompileError", "Expected 2 arguments but found 3.", [0]) + js: err("ReqlCompileError", "Expected 1 argument (not including options) but found 2.", [0]) + + # r.js() + - cd: r.js('1 + 1') + ot: 2 + + - cd: r.js('1 + 1; 2 + 2') + ot: 4 + + - cd: r.do(1, 2, r.js('(function(a, b) { return a + b; })')) + ot: 3 + + - cd: r.expr(1).do(r.js('(function(x) { return x + 1; })')) + ot: 2 + + - cd: r.expr('foo').do(r.js('(function(x) { return x + "bar"; })')) + ot: 'foobar' + + # js timeout optarg shouldn't be triggered + - cd: r.js('1 + 2', {timeout:1.2}) + py: r.js('1 + 2', timeout=1.2) + ot: 3 + + # js error cases + - cd: r.js('(function() { return 1; })') + ot: err("ReqlQueryLogicError", "Query result must be of type DATUM, GROUPED_DATA, or STREAM (got FUNCTION).", [0]) + + - cd: r.js('function() { return 1; }') + ot: err("ReqlQueryLogicError", "SyntaxError: Unexpected token (", [0]) + + # Play with the number of arguments in the JS function + - cd: r.do(1, 2, r.js('(function(a) { return a; })')) + ot: 1 + + - cd: r.do(1, 2, r.js('(function(a, b, c) { return a; })')) + ot: 1 + + - cd: r.do(1, 2, r.js('(function(a, b, c) { return c; })')) + ot: err("ReqlQueryLogicError", "Cannot convert javascript `undefined` to ql::datum_t.", [0]) + + - cd: r.expr([1, 2, 3]).filter(r.js('(function(a) { return a >= 2; })')) + ot: ([2, 3]) + + - cd: r.expr([1, 2, 3]).map(r.js('(function(a) { return a + 1; })')) + ot: ([2, 3, 4]) + + - cd: r.expr([1, 2, 3]).map(r.js('1')) + ot: err("ReqlQueryLogicError", "Expected type FUNCTION but found DATUM:", [0]) + + - cd: r.expr([1, 2, 3]).filter(r.js('(function(a) {})')) + ot: err("ReqlQueryLogicError", "Cannot convert javascript `undefined` to ql::datum_t.", [0]) + + # What happens if we pass static values to things that expect functions + - cd: r.expr([1, 2, 3]).map(1) + ot: err("ReqlQueryLogicError", "Expected type FUNCTION but found DATUM:", [0]) + + - cd: r.expr([1, 2, 3]).filter('foo') + ot: ([1, 2, 3]) + - cd: r.expr([1, 2, 4]).filter([]) + ot: ([1, 2, 4]) + - cd: r.expr([1, 2, 3]).filter(null) + ot: ([]) + + - cd: r.expr([1, 2, 4]).filter(False) + rb: r([1, 2, 4]).filter(false) + ot: ([]) + + # forEach + - cd: tbl.count() + ot: 0 + + # Insert three elements + - js: r([1, 2, 3]).forEach(function (row) { return tbl.insert({ id:row }) }) + py: r.expr([1, 2, 3]).for_each(lambda row:tbl.insert({ 'id':row })) + rb: r([1, 2, 3]).for_each{ |row| tbl.insert({ :id => row }) } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':3}) + + - cd: tbl.count() + ot: 3 + + # Update each row to add additional attribute + - js: r([1, 2, 3]).forEach(function (row) { return tbl.update({ foo:row }) }) + py: r.expr([1,2,3]).for_each(lambda row:tbl.update({'foo':row})) + rb: r.expr([1,2,3]).for_each{ |row| tbl.update({ :foo => row }) } + ot: ({'deleted':0.0,'replaced':9,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # Insert three more elements (and error on three) + - js: r([1, 2, 3]).forEach(function (row) { return [tbl.insert({ id:row }), tbl.insert({ id:row.mul(10) })] }) + py: r.expr([1,2,3]).for_each(lambda row:[tbl.insert({ 'id':row }), tbl.insert({ 'id':row*10 })]) + rb: r.expr([1,2,3]).for_each{ |row| [tbl.insert({ :id => row}), tbl.insert({ :id => row*10})] } + ot: {'first_error':"Duplicate primary key `id`:\n{\n\t\"foo\":\t3,\n\t\"id\":\t1\n}\n{\n\t\"id\":\t1\n}",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':3,'skipped':0.0,'inserted':3} + + - cd: tbl.count() + ot: 6 + + - cd: tableCount = tbl2.count() + - cd: r.expr([1, 2, 3]).for_each( tbl2.insert({}) ) + ot: ({'deleted':0.0,'replaced':0.0,'generated_keys':arrlen(3,uuid()),'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':3}) + # inserts only a single document per #3700 + - cd: tbl2.count() + ot: tableCount + 1 + + # We have six elements, update them 6*2*3=36 times + - js: r([1, 2, 3]).forEach(function (row) { return [tbl.update({ foo:row }), tbl.update({ bar:row })] }) + py: r.expr([1,2,3]).for_each(lambda row:[tbl.update({'foo':row}), tbl.update({'bar':row})]) + rb: r.expr([1,2,3]).for_each{ |row| [tbl.update({:foo => row}), tbl.update({:bar => row})]} + ot: ({'deleted':0.0,'replaced':36,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # forEach negative cases + - cd: r.expr([1, 2, 3]).for_each( tbl2.insert({ 'id':r.row }) ) + rb: r([1, 2, 3]).for_each{ |row| tbl2.insert({ 'id':row }) } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':3}) + + - cd: r.expr([1, 2, 3]).for_each(1) + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries. Expected type ARRAY but found NUMBER.", [0]) + + - py: r.expr([1, 2, 3]).for_each(lambda x:x) + js: r([1, 2, 3]).forEach(function (x) { return x; }) + rb: r([1, 2, 3]).for_each{ |x| x } + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries. Expected type ARRAY but found NUMBER.", [1, 1]) + + - cd: r.expr([1, 2, 3]).for_each(r.row) + rb: r([1, 2, 3]).for_each{ |row| row } + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries. Expected type ARRAY but found NUMBER.", [1, 1]) + + - js: r([1, 2, 3]).forEach(function (row) { return tbl; }) + py: r.expr([1, 2, 3]).for_each(lambda row:tbl) + rb: r([1, 2, 3]).for_each{ |row| tbl } + ot: err("ReqlQueryLogicError", "FOR_EACH expects one or more basic write queries.", [1, 1]) + + # This is only relevant in JS -- what happens when we return undefined + - js: r([1, 2, 3]).forEach(function (row) {}) + ot: err("ReqlDriverCompileError", 'Anonymous function returned `undefined`. Did you forget a `return`?', [1]) + + # Make sure write queries can't be nested into stream ops + - cd: r.expr(1).do(tbl.insert({'foo':r.row})) + rb: r(1).do{ |row| tbl.insert({ :foo => row }) } + ot: ({'deleted':0.0,'replaced':0.0,'generated_keys':arrlen(1,uuid()),'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':1}) + + - py: r.expr([1, 2])[0].do(tbl.insert({'foo':r.row})) + js: r.expr([1, 2]).nth(0).do(tbl.insert({'foo':r.row})) + rb: r([1, 2])[0].do{ |row| tbl.insert({ :foo => row }) } + ot: ({'deleted':0.0,'replaced':0.0,'generated_keys':arrlen(1,uuid()),'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':1}) + + - cd: r.expr([1, 2]).map(tbl.insert({'foo':r.row})) + rb: r([1, 2]).map{ |row| tbl.insert({ :foo => row }) } + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr([1, 2]).map(r.db('test').table_create('table_create_failure')) + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr([1, 2]).map(tbl.insert({'foo':r.row}).get_field('inserted')) + rb: r.expr([1, 2]).map{|x| tbl.insert({'foo':x}).get_field('inserted')} + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr([1, 2]).map(tbl.insert({'foo':r.row}).get_field('inserted').add(5)) + rb: r.expr([1, 2]).map{|x| tbl.insert({'foo':x}).get_field('inserted').add(5)} + ot: err('ReqlCompileError', 'Cannot nest writes or meta ops in stream operations. Use FOR_EACH instead.', [0]) + + - cd: r.expr(1).do(r.db('test').table_create('table_create_success')) + ot: partial({'tables_created':1}) diff --git a/ext/librethinkdbxx/test/upstream/datum/array.yaml b/ext/librethinkdbxx/test/upstream/datum/array.yaml new file mode 100644 index 00000000..7d16f943 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/array.yaml @@ -0,0 +1,133 @@ +desc: Tests conversion to and from the RQL array type +tests: + - cd: + - r.expr([]) + - r([]) + py: r.expr([]) + ot: [] + + - py: r.expr([1]) + js: r([1]) + rb: r([1]) + ot: [1] + + - py: r.expr([1,2,3,4,5]) + js: r([1,2,3,4,5]) + rb: r.expr([1,2,3,4,5]) + ot: [1,2,3,4,5] + + - cd: r.expr([]).type_of() + ot: 'ARRAY' + + # test coercions + - cd: + - r.expr([1, 2]).coerce_to('string') + - r.expr([1, 2]).coerce_to('STRING') + ot: '[1,2]' + + - cd: r.expr([1, 2]).coerce_to('array') + ot: [1, 2] + + - cd: r.expr([1, 2]).coerce_to('number') + ot: err('ReqlQueryLogicError', 'Cannot coerce ARRAY to NUMBER.', [0]) + + - cd: r.expr([['a', 1], ['b', 2]]).coerce_to('object') + ot: {'a':1,'b':2} + + - cd: r.expr([[]]).coerce_to('object') + ot: err('ReqlQueryLogicError', 'Expected array of size 2, but got size 0.') + + - cd: r.expr([['1',2,3]]).coerce_to('object') + ot: err('ReqlQueryLogicError', 'Expected array of size 2, but got size 3.') + + # Nested expression + - cd: r.expr([r.expr(1)]) + ot: [1] + + - cd: r.expr([1,3,4]).insert_at(1, 2) + ot: [1,2,3,4] + - cd: r.expr([2,3]).insert_at(0, 1) + ot: [1,2,3] + - cd: r.expr([1,2,3]).insert_at(-1, 4) + ot: [1,2,3,4] + - cd: r.expr([1,2,3]).insert_at(3, 4) + ot: [1,2,3,4] + - py: r.expr(3).do(lambda x: r.expr([1,2,3]).insert_at(x, 4)) + - js: r.expr(3).do(function (x) { return r.expr([1,2,3]).insert_at(x, 4); }) + - rb: r.expr(3).do{|x| r.expr([1,2,3]).insert_at(x, 4)} + ot: [1,2,3,4] + - cd: r.expr([1,2,3]).insert_at(4, 5) + ot: err('ReqlNonExistenceError', 'Index `4` out of bounds for array of size: `3`.', [0]) + - cd: r.expr([1,2,3]).insert_at(-5, -1) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -5', [0]) + - cd: r.expr([1,2,3]).insert_at(1.5, 1) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).insert_at(null, 1) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) + + - cd: r.expr([1,4]).splice_at(1, [2,3]) + ot: [1,2,3,4] + - cd: r.expr([3,4]).splice_at(0, [1,2]) + ot: [1,2,3,4] + - cd: r.expr([1,2]).splice_at(2, [3,4]) + ot: [1,2,3,4] + - cd: r.expr([1,2]).splice_at(-1, [3,4]) + ot: [1,2,3,4] + - py: r.expr(2).do(lambda x: r.expr([1,2]).splice_at(x, [3,4])) + - js: r.expr(2).do(function (x) { return r.expr([1,2]).splice_at(x, [3,4]); }) + - rb: r.expr(2).do{|x| r.expr([1,2]).splice_at(x, [3,4])} + ot: [1,2,3,4] + - cd: r.expr([1,2]).splice_at(3, [3,4]) + ot: err('ReqlNonExistenceError', 'Index `3` out of bounds for array of size: `2`.', [0]) + - cd: r.expr([1,2]).splice_at(-4, [3,4]) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -4', [0]) + - cd: r.expr([1,2,3]).splice_at(1.5, [1]) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).splice_at(null, [1]) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) + - cd: r.expr([1,4]).splice_at(1, 2) + ot: err('ReqlQueryLogicError', 'Expected type ARRAY but found NUMBER.', [0]) + + - cd: r.expr([1,2,3,4]).delete_at(0) + ot: [2,3,4] + - py: r.expr(0).do(lambda x: r.expr([1,2,3,4]).delete_at(x)) + - js: r.expr(0).do(function (x) { return r.expr([1,2,3,4]).delete_at(x); }) + - rb: r.expr(0).do{|x| r.expr([1,2,3,4]).delete_at(x)} + ot: [2,3,4] + - cd: r.expr([1,2,3,4]).delete_at(-1) + ot: [1,2,3] + - cd: r.expr([1,2,3,4]).delete_at(1,3) + ot: [1,4] + - cd: r.expr([1,2,3,4]).delete_at(4,4) + ot: [1,2,3,4] + - cd: r.expr([]).delete_at(0,0) + ot: [] + - cd: r.expr([1,2,3,4]).delete_at(1,-1) + ot: [1,4] + - cd: r.expr([1,2,3,4]).delete_at(4) + ot: err('ReqlNonExistenceError', 'Index `4` out of bounds for array of size: `4`.', [0]) + - cd: r.expr([1,2,3,4]).delete_at(-5) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -5', [0]) + - cd: r.expr([1,2,3]).delete_at(1.5) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).delete_at(null) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) + + - cd: r.expr([0,2,3]).change_at(0, 1) + ot: [1,2,3] + - py: r.expr(1).do(lambda x: r.expr([0,2,3]).change_at(0,x)) + - js: r.expr(1).do(function (x) { return r.expr([0,2,3]).change_at(0,x); }) + - rb: r.expr(1).do{|x| r.expr([0,2,3]).change_at(0,x)} + ot: [1,2,3] + - cd: r.expr([1,0,3]).change_at(1, 2) + ot: [1,2,3] + - cd: r.expr([1,2,0]).change_at(2, 3) + ot: [1,2,3] + - cd: r.expr([1,2,3]).change_at(3, 4) + ot: err('ReqlNonExistenceError', 'Index `3` out of bounds for array of size: `3`.', [0]) + - cd: r.expr([1,2,3,4]).change_at(-5, 1) + ot: err('ReqlNonExistenceError', 'Index out of bounds: -5', [0]) + - cd: r.expr([1,2,3]).change_at(1.5, 1) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) + - cd: r.expr([1,2,3]).change_at(null, 1) + ot: err('ReqlNonExistenceError', 'Expected type NUMBER but found NULL.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/datum/binary.yaml b/ext/librethinkdbxx/test/upstream/datum/binary.yaml new file mode 100644 index 00000000..6236a457 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/binary.yaml @@ -0,0 +1,363 @@ +desc: Tests of converstion to and from the RQL binary type +tests: + + # Short binary data from 0 to 12 characters + # Not fully implemented for JS as comparing Buffer objects is non-trivial + - def: + rb: s = "".force_encoding('BINARY') + py: s = b'' + js: s = Buffer("", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 0 + + - def: + rb: s = "\x00".force_encoding('BINARY') + py: s = b'\x00' + js: s = Buffer("\x00", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 1 + + - def: + rb: s = "\x00\x42".force_encoding('BINARY') + py: s = b'\x00\x42' + js: s = Buffer("\x00\x42", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 2 + + - def: + rb: s = "\x00\xfe\x7a".force_encoding('BINARY') + py: s = b'\x00\xfe\x7a' + js: s = Buffer("\x00\xfe\x7a", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 3 + + - def: + rb: s = "\xed\xfe\x00\xba".force_encoding('BINARY') + py: s = b'\xed\xfe\x00\xba' + js: s = Buffer("\xed\xfe\x00\xba", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 4 + + - def: + rb: s = "\x50\xf9\x00\x77\xf9".force_encoding('BINARY') + py: s = b'\x50\xf9\x00\x77\xf9' + js: s = Buffer("\x50\xf9\x00\x77\xf9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 5 + + - def: + rb: s = "\x2f\xe3\xb5\x57\x00\x92".force_encoding('BINARY') + py: s = b'\x2f\xe3\xb5\x57\x00\x92' + js: s = Buffer("\x2f\xe3\xb5\x57\x00\x92", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 6 + + - def: + rb: s = "\xa9\x43\x54\xe9\x00\xf8\xfb".force_encoding('BINARY') + py: s = b'\xa9\x43\x54\xe9\x00\xf8\xfb' + js: s = Buffer("\xa9\x43\x54\xe9\x00\xf8\xfb", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 7 + + - def: + rb: s = "\x57\xbb\xe5\x82\x8b\xd3\x00\xf9".force_encoding('BINARY') + py: s = b'\x57\xbb\xe5\x82\x8b\xd3\x00\xf9' + js: s = Buffer("\x57\xbb\xe5\x82\x8b\xd3\x00\xf9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 8 + + - def: + rb: s = "\x44\x1b\x3e\x00\x13\x19\x29\x2a\xbf".force_encoding('BINARY') + py: s = b'\x44\x1b\x3e\x00\x13\x19\x29\x2a\xbf' + js: s = Buffer("\x44\x1b\x3e\x00\x13\x19\x29\x2a\xbf", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 9 + + - def: + rb: s = "\x8a\x1d\x09\x00\x5d\x60\x6b\x2e\x70\xd9".force_encoding('BINARY') + py: s = b'\x8a\x1d\x09\x00\x5d\x60\x6b\x2e\x70\xd9' + js: s = Buffer("\x8a\x1d\x09\x00\x5d\x60\x6b\x2e\x70\xd9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 10 + + - def: + rb: s = "\x00\xaf\x47\x4b\x38\x99\x14\x8d\x8f\x10\x51".force_encoding('BINARY') + py: s = b'\x00\xaf\x47\x4b\x38\x99\x14\x8d\x8f\x10\x51' + js: s = Buffer("\x00\xaf\x47\x4b\x38\x99\x14\x8d\x8f\x10\x51", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 11 + + - def: + cd: s = "\x45\x39\x00\xf7\xc2\x37\xfd\xe0\x38\x82\x40\xa9".force_encoding('BINARY') + py: s = b'\x45\x39\x00\xf7\xc2\x37\xfd\xe0\x38\x82\x40\xa9' + js: s = Buffer("\x45\x39\x00\xf7\xc2\x37\xfd\xe0\x38\x82\x40\xa9", 'binary') + - cd: r.binary(s) + ot: s + - cd: r.binary(s).count() + ot: 12 + + # Test comparisons + # Binary objects to use, in order of increasing value + - def: + js: a = Buffer("\x00", 'binary') + rb: a = "\x00".force_encoding('BINARY') + py: a = b'\x00' + - def: + js: b = Buffer("\x00\x01", 'binary') + rb: b = "\x00\x01".force_encoding('BINARY') + py: b = b'\x00\x01' + - def: + js: c = Buffer("\x01", 'binary') + rb: c = "\x01".force_encoding('BINARY') + py: c = b'\x01' + - def: + js: d = Buffer("\x70\x22", 'binary') + rb: d = "\x70\x22".force_encoding('BINARY') + py: d = b'\x70\x22' + - def: + js: e = Buffer("\x80", 'binary') + rb: e = "\x80".force_encoding('BINARY') + py: e = b'\x80' + - def: + js: f = Buffer("\xFE", 'binary') + rb: f = "\xFE".force_encoding('BINARY') + py: f = b'\xFE' + + # a -> a + - cd: r.binary(a).eq(r.binary(a)) + ot: true + - cd: r.binary(a).le(r.binary(a)) + ot: true + - cd: r.binary(a).ge(r.binary(a)) + ot: true + - cd: r.binary(a).ne(r.binary(a)) + ot: false + - cd: r.binary(a).lt(r.binary(a)) + ot: false + - cd: r.binary(a).gt(r.binary(a)) + ot: false + + # a -> b + - cd: r.binary(a).ne(r.binary(b)) + ot: true + - cd: r.binary(a).lt(r.binary(b)) + ot: true + - cd: r.binary(a).le(r.binary(b)) + ot: true + - cd: r.binary(a).ge(r.binary(b)) + ot: false + - cd: r.binary(a).gt(r.binary(b)) + ot: false + - cd: r.binary(a).eq(r.binary(b)) + ot: false + + # b -> c + - cd: r.binary(b).ne(r.binary(c)) + ot: true + - cd: r.binary(b).lt(r.binary(c)) + ot: true + - cd: r.binary(b).le(r.binary(c)) + ot: true + - cd: r.binary(b).ge(r.binary(c)) + ot: false + - cd: r.binary(b).gt(r.binary(c)) + ot: false + - cd: r.binary(b).eq(r.binary(c)) + ot: false + + # c -> d + - cd: r.binary(c).ne(r.binary(d)) + ot: true + - cd: r.binary(c).lt(r.binary(d)) + ot: true + - cd: r.binary(c).le(r.binary(d)) + ot: true + - cd: r.binary(c).ge(r.binary(d)) + ot: false + - cd: r.binary(c).gt(r.binary(d)) + ot: false + - cd: r.binary(c).eq(r.binary(d)) + ot: false + + # d -> e + - cd: r.binary(d).ne(r.binary(e)) + ot: true + - cd: r.binary(d).lt(r.binary(e)) + ot: true + - cd: r.binary(d).le(r.binary(e)) + ot: true + - cd: r.binary(d).ge(r.binary(e)) + ot: false + - cd: r.binary(d).gt(r.binary(e)) + ot: false + - cd: r.binary(d).eq(r.binary(e)) + ot: false + + # e -> f + - cd: r.binary(e).ne(r.binary(f)) + ot: true + - cd: r.binary(e).lt(r.binary(f)) + ot: true + - cd: r.binary(e).le(r.binary(f)) + ot: true + - cd: r.binary(e).ge(r.binary(f)) + ot: false + - cd: r.binary(e).gt(r.binary(f)) + ot: false + - cd: r.binary(e).eq(r.binary(f)) + ot: false + + # f -> f + - cd: r.binary(f).eq(r.binary(f)) + ot: true + - cd: r.binary(f).le(r.binary(f)) + ot: true + - cd: r.binary(f).ge(r.binary(f)) + ot: true + - cd: r.binary(f).ne(r.binary(f)) + ot: false + - cd: r.binary(f).lt(r.binary(f)) + ot: false + - cd: r.binary(f).gt(r.binary(f)) + ot: false + + # Test encodings + - py: + cd: r.binary(u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム'.encode('utf-8')) + ot: u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム'.encode('utf-8') + py3: + cd: r.binary(str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム').encode('utf-8')) + ot: str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム').encode('utf-8') + - py: + cd: r.binary(u'ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ'.encode('utf-16')) + ot: u'ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ'.encode('utf-16') + py3: + cd: r.binary(str('ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ').encode('utf-16')) + ot: str('ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏ').encode('utf-16') + - py: + cd: r.binary(u'lorem ipsum'.encode('ascii')) + ot: u'lorem ipsum'.encode('ascii') + py3: + cd: r.binary(str('lorem ipsum').encode('ascii')) + ot: str('lorem ipsum').encode('ascii') + + # Test coercions + - py: r.binary(b'foo').coerce_to('string') + ot: 'foo' + - py: + cd: r.binary(u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム'.encode('utf-8')).coerce_to('string') + ot: u'イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム' + py3: + cd: r.binary(str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム').encode('utf-8')).coerce_to('string') + ot: str('イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム') + - py: + cd: r.binary(u'lorem ipsum'.encode('ascii')).coerce_to('string') + ot: u'lorem ipsum' + py3: + cd: r.binary(str('lorem ipsum').encode('ascii')).coerce_to('string') + ot: str('lorem ipsum') + + - py: r.expr('foo').coerce_to('binary') + ot: b'foo' + + - cd: r.binary(a).coerce_to('bool') + ot: True + + - py: r.binary(b'foo').coerce_to('binary') + ot: b'foo' + + # Test slice + - py: r.binary(b'abcdefg').slice(-3,-1) + ot: b'ef' + - py: r.binary(b'abcdefg').slice(0, 2) + ot: b'ab' + - py: r.binary(b'abcdefg').slice(3, -1) + ot: b'def' + - py: r.binary(b'abcdefg').slice(-5, 5) + ot: b'cde' + - py: r.binary(b'abcdefg').slice(-8, 2) + ot: b'ab' + - py: r.binary(b'abcdefg').slice(5, 7) + ot: b'fg' + + # Left side out-of-bound should clamp to index 0 + - py: r.binary(b'abcdefg').slice(-9, 2) + ot: b'ab' + + # Right side out-of-bound should return the valid subset of the range + - py: r.binary(b'abcdefg').slice(5, 9) + ot: b'fg' + + # Test binary_format optarg + - cd: r.binary(b) + runopts: + binary_format: "native" + ot: b + - cd: r.binary(b) + runopts: + binary_format: "raw" + ot: {'$reql_type$':'BINARY','data':'AAE='} + + # Test r.binary of nested terms + - cd: r.binary(r.expr("data")) + ot: + js: Buffer("data", "binary") + rb: "data" + py: b"data" + + - cd: r.binary(r.expr({})) + ot: err('ReqlQueryLogicError', 'Expected type STRING but found OBJECT.', []) + + - cd: r.binary(r.expr([])) + ot: err('ReqlQueryLogicError', 'Expected type STRING but found ARRAY.', []) + + # Test errors + + # Missing 'data' field + - py: r.expr({'$reql_type$':'BINARY'}) + rb: r.expr({'$reql_type$':'BINARY'}) + ot: err('ReqlQueryLogicError','Invalid binary pseudotype:'+' lacking `data` key.',[]) + + # Invalid base64 format + - py: r.expr({'$reql_type$':'BINARY','data':'ABCDEFGH==AA'}) + ot: err('ReqlQueryLogicError','Invalid base64 format, data found after padding character \'=\'.',[]) + - py: r.expr({'$reql_type$':'BINARY','data':'ABCDEF==$'}) + ot: err('ReqlQueryLogicError','Invalid base64 format, data found after padding character \'=\'.',[]) + - py: r.expr({'$reql_type$':'BINARY','data':'A^CDEFGH'}) + ot: err('ReqlQueryLogicError','Invalid base64 character found:'+' \'^\'.',[]) + - py: r.expr({'$reql_type$':'BINARY','data':'ABCDE'}) + ot: err('ReqlQueryLogicError','Invalid base64 length:'+' 1 character remaining, cannot decode a full byte.',[]) + + # Invalid coercions + - cd: r.binary(a).coerce_to('array') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to ARRAY.',[]) + - cd: r.binary(a).coerce_to('object') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to OBJECT.',[]) + - cd: r.binary(a).coerce_to('number') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to NUMBER.',[]) + - cd: r.binary(a).coerce_to('nu'+'ll') + ot: err('ReqlQueryLogicError','Cannot coerce BINARY to NULL.',[]) diff --git a/ext/librethinkdbxx/test/upstream/datum/bool.yaml b/ext/librethinkdbxx/test/upstream/datum/bool.yaml new file mode 100644 index 00000000..0e0aa11f --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/bool.yaml @@ -0,0 +1,47 @@ +desc: Tests of conversion to and from the RQL bool type +tests: + - py: r.expr(True) + js: + - r.expr(true) + - r(true) + rb: r true + ot: true + + - py: r.expr(False) + js: + - r.expr(false) + - r(false) + rb: r false + ot: false + + - cd: r.expr(False).type_of() + ot: 'BOOL' + + # test coercions + - cd: r.expr(True).coerce_to('string') + ot: 'true' + + - cd: r.expr(True).coerce_to('bool') + ot: True + + - cd: r.expr(False).coerce_to('bool') + ot: False + + - cd: r.expr(null).coerce_to('bool') + ot: False + + - cd: r.expr(0).coerce_to('bool') + ot: True + + - cd: r.expr('false').coerce_to('bool') + ot: True + + - cd: r.expr('foo').coerce_to('bool') + ot: True + + - cd: r.expr([]).coerce_to('bool') + ot: True + + - cd: r.expr({}).coerce_to('bool') + ot: True + diff --git a/ext/librethinkdbxx/test/upstream/datum/null.yaml b/ext/librethinkdbxx/test/upstream/datum/null.yaml new file mode 100644 index 00000000..f8e95464 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/null.yaml @@ -0,0 +1,18 @@ +desc: Tests of conversion to and from the RQL null type +tests: + - cd: + - r(null) + - r.expr(null) + py: r.expr(null) + ot: (null) + + - cd: r.expr(null).type_of() + rb: r(null).type_of() + ot: 'NULL' + + # test coercions + - cd: r.expr(null).coerce_to('string') + ot: 'null' + + - cd: r.expr(null).coerce_to('null') + ot: null diff --git a/ext/librethinkdbxx/test/upstream/datum/number.yaml b/ext/librethinkdbxx/test/upstream/datum/number.yaml new file mode 100644 index 00000000..09c48d15 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/number.yaml @@ -0,0 +1,125 @@ +# desc will be included in a comment to help identify test groups +desc: Tests of conversion to and from the RQL number type +tests: + + # Simple integers + - cd: r.expr(1) + js: + - r(1) + - r.expr(1) + rb: + - r 1 + - r(1) + - r.expr(1) + ot: 1 + - cd: r.expr(-1) + js: + - r(-1) + - r.expr(-1) + rb: + - r -1 + - r(-1) + - r.expr(-1) + ot: -1 + - cd: r.expr(0) + js: + - r(0) + - r.expr(0) + rb: + - r 0 + - r(0) + - r.expr(0) + ot: 0 + + # Floats + - cd: r.expr(1.0) + js: + - r(1.0) + - r.expr(1.0) + rb: + - r 1.0 + - r(1.0) + - r.expr(1.0) + ot: 1.0 + - cd: r.expr(1.5) + js: + - r(1.5) + - r.expr(1.5) + rb: + - r 1.5 + - r(1.5) + - r.expr(1.5) + ot: 1.5 + - cd: r.expr(-0.5) + js: + - r(-0.5) + - r.expr(-0.5) + rb: + - r -0.5 + - r(-0.5) + - r.expr(-0.5) + ot: -0.5 + - cd: r.expr(67498.89278) + js: + - r(67498.89278) + - r.expr(67498.89278) + rb: + - r 67498.89278 + - r(67498.89278) + - r.expr(67498.89278) + ot: 67498.89278 + + # Big numbers + - cd: r.expr(1234567890) + js: + - r(1234567890) + - r.expr(1234567890) + rb: + - r 1234567890 + - r(1234567890) + - r.expr(1234567890) + ot: 1234567890 + + - cd: r.expr(-73850380122423) + js: + - r.expr(-73850380122423) + - r(-73850380122423) + rb: + - r -73850380122423 + - r.expr(-73850380122423) + - r(-73850380122423) + ot: -73850380122423 + + # Test that numbers round-trip correctly + - py: + cd: r.expr(1234567890123456789012345678901234567890) + ot: float(1234567890123456789012345678901234567890) + js: + cd: r.expr(1234567890123456789012345678901234567890) + ot: 1234567890123456789012345678901234567890 + - cd: r.expr(123.4567890123456789012345678901234567890) + ot: 123.4567890123456789012345678901234567890 + + - cd: r.expr(1).type_of() + ot: 'NUMBER' + + # test coercions + - cd: r.expr(1).coerce_to('string') + ot: '1' + + - cd: r.expr(1).coerce_to('number') + ot: 1 + + # The drivers now convert to an int (where relevant) if we think the result + # looks like an int (result % 1.0 == 0.0) + - py: r.expr(1.0) + rb: r 1.0 + ot: int_cmp(1) + + - py: r.expr(45) + rb: r 45 + ot: int_cmp(45) + + - py: r.expr(1.2) + rb: r 1.2 + ot: float_cmp(1.2) diff --git a/ext/librethinkdbxx/test/upstream/datum/object.yaml b/ext/librethinkdbxx/test/upstream/datum/object.yaml new file mode 100644 index 00000000..64b3cbf4 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/object.yaml @@ -0,0 +1,85 @@ +desc: Tests conversion to and from the RQL object type +tests: + - cd: + - r({}) + - r.expr({}) + py: r.expr({}) + ot: {} + - cd: + - r({a:1}) + - r.expr({'a':1}) + py: r.expr({'a':1}) + ot: {'a':1} + - cd: + - r({a:1, b:'two', c:True}) + - r.expr({'a':1, 'b':'two', 'c':True}) + py: r.expr({'a':1, 'b':'two', 'c':True}) + ot: {'a':1, 'b':'two', 'c':True} + + # Nested expressions + - cd: r.expr({'a':r.expr(1)}) + ot: {'a':1} + + - cd: r.expr({'a':{'b':[{'c':2}, 'a', 4]}}) + ot: {'a':{'b':[{'c':2}, 'a', 4]}} + + - cd: r.expr({'a':1}).type_of() + ot: 'OBJECT' + + # test coercions + - cd: r.expr({'a':1}).coerce_to('string') + ot: + cd: '{"a":1}' + + - cd: r.expr({'a':1}).coerce_to('object') + ot: {'a':1} + + - cd: r.expr({'a':1}).coerce_to('array') + ot: [['a',1]] + + # Error cases + - cd: r.expr({12:'a'}) + # JavaScript auto-converts keys for us + js: + ot: err_regex("ReqlCompileError", "Object keys must be strings.*") + + - cd: r.expr({'a':{12:'b'}}) + # JavaScript auto-converts keys for us + js: + ot: err_regex("ReqlCompileError", "Object keys must be strings.*") + + - js: r({'a':undefined}) + ot: err("ReqlCompileError", "Object field 'a' may not be undefined") + + - js: r({'a':{'b':undefined}}) + ot: err("ReqlCompileError", "Object field 'b' may not be undefined") + + - cd: r.expr({}, "foo") + ot: + cd: err("ReqlCompileError", "Second argument to `r.expr` must be a number.") + js: err("ReqlCompileError", "Second argument to `r.expr` must be a number or undefined.") + + - js: r.expr({}, NaN) + ot: err("ReqlCompileError", "Second argument to `r.expr` must be a number or undefined.") + + # r.object + - cd: r.object() + ot: {} + + - cd: r.object('a', 1, 'b', 2) + ot: {'a':1,'b':2} + + - cd: r.object('c'+'d', 3) + ot: {'cd':3} + + - cd: r.object('o','d','d') + ot: err("ReqlQueryLogicError", "OBJECT expects an even number of arguments (but found 3).", []) + + - cd: r.object(1, 1) + ot: err("ReqlQueryLogicError","Expected type STRING but found NUMBER.",[]) + + - cd: r.object('e', 4, 'e', 5) + ot: err("ReqlQueryLogicError","Duplicate key \"e\" in object. (got 4 and 5 as values)",[]) + + - cd: r.object('g', r.db('test')) + ot: err("ReqlQueryLogicError","Expected type DATUM but found DATABASE:",[]) diff --git a/ext/librethinkdbxx/test/upstream/datum/string.yaml b/ext/librethinkdbxx/test/upstream/datum/string.yaml new file mode 100644 index 00000000..c518175d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/string.yaml @@ -0,0 +1,329 @@ +desc: Tests of converstion to and from the RQL string type +tests: + + - def: + cd: japanese_hello = 'こんにちは' + # Python supports unicode strings with the u'' pattern, except 3.0-3.2 + py: japanese_hello = u'こんにちは' + py3.0: japanese_hello = 'こんにちは' + py3.1: japanese_hello = 'こんにちは' + py3.2: japanese_hello = 'こんにちは' + + # Simple strings + - cd: + - r('str') + - r.expr('str') + py: r.expr('str') + ot: "str" + - cd: + - r("str") + - r.expr("str") + py: r.expr("str") + ot: "str" + + # Unicode + + - cd: + py: + cd: r.expr(u'str') + ot: u'str' + py3.0: r.expr('str') + py3.1: r.expr('str') + py3.2: r.expr('str') + ot: 'str' + + - cd: r.expr(japanese_hello) + ot: + cd: 'こんにちは' + py: u'こんにちは' + py3.0: 'こんにちは' + py3.1: 'こんにちは' + py3.2: 'こんにちは' + + - cd: r.expr('foo').type_of() + ot: 'STRING' + + # test coercions + - cd: r.expr('foo').coerce_to('string') + ot: 'foo' + - cd: r.expr('-1.2').coerce_to('NUMBER') + ot: -1.2 + - cd: r.expr('--1.2').coerce_to('NUMBER') + ot: err("ReqlQueryLogicError", "Could not coerce `--1.2` to NUMBER.", []) + - cd: r.expr('-1.2-').coerce_to('NUMBER') + ot: err("ReqlQueryLogicError", "Could not coerce `-1.2-` to NUMBER.", []) + - cd: r.expr('0xa').coerce_to('NUMBER') + ot: 10 + - cd: r.expr('inf').coerce_to('NUMBER') + ot: err("ReqlQueryLogicError", "Non-finite number: inf", []) + + # count is defined as the number of unicode codepoints + - cd: r.expr('hello, world!').count() + ot: 13 + - cd: r.expr(japanese_hello).count() + ot: 5 + + # slice is defined on unicode codepoints + - cd: r.expr('hello').slice(1) + ot: 'ello' + - cd: r.expr('hello').slice(-1) + ot: 'o' + - cd: r.expr('hello').slice(-4,3) + ot: 'el' + - cd: r.expr('hello').slice(-99) + ot: 'hello' + - cd: r.expr('hello').slice(0) + ot: 'hello' + - cd: r.expr(japanese_hello).slice(1) + ot: + cd: 'んにちは' + py: u'んにちは' + py3.0: 'んにちは' + py3.1: 'んにちは' + py3.2: 'んにちは' + - cd: r.expr(japanese_hello).slice(1,2) + ot: + cd: 'ん' + py: u'ん' + py3.0: 'ん' + py3.1: 'ん' + py3.2: 'ん' + - cd: r.expr(japanese_hello).slice(-3) + ot: + cd: 'にちは' + py: u'にちは' + py3.0: 'にちは' + py3.1: 'にちは' + py3.2: 'にちは' + + # This is how these edge cases are handled in Python. + - cd: r.expr('').split() + ot: [] + - cd: r.expr('').split(null) + ot: [] + - cd: r.expr('').split(' ') + ot: [''] + - cd: r.expr('').split('') + ot: [] + - cd: r.expr('').split(null, 5) + ot: [] + - cd: r.expr('').split(' ', 5) + ot: [''] + - cd: r.expr('').split('', 5) + ot: [] + + - cd: r.expr('aaaa bbbb cccc ').split() + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr('aaaa bbbb cccc ').split(null) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr('aaaa bbbb cccc ').split(' ') + ot: ['aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr('aaaa bbbb cccc ').split('') + ot: ['a', 'a', 'a', 'a', ' ', 'b', 'b', 'b', 'b', ' ', ' ', 'c', 'c', 'c', 'c', ' '] + - cd: r.expr('aaaa bbbb cccc ').split('b') + ot: ['aaaa ', '', '', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb') + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ') + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb') + ot: ['aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr('aaaa bbbb cccc ').split(null, 3) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr('aaaa bbbb cccc ').split(' ', 5) + ot: ['aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr('aaaa bbbb cccc ').split('', 5) + ot: ['a', 'a', 'a', 'a', ' ', 'bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('b', 5) + ot: ['aaaa ', '', '', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb', 3) + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ', 2) + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb', 6) + ot: ['aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 3) + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr('aaaa bbbb cccc ').split(null, 2) + ot: ['aaaa', 'bbbb', 'cccc '] + - cd: r.expr("a b ").split(null, 2) + ot: ["a", "b"] + - cd: r.expr('aaaa bbbb cccc ').split(' ', 4) + ot: ['aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr('aaaa bbbb cccc ').split('', 4) + ot: ['a', 'a', 'a', 'a', ' bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('b', 4) + ot: ['aaaa ', '', '', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb', 2) + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ', 1) + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb', 5) + ot: ['aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 1) + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr('aaaa bbbb cccc ').split(null, 1) + ot: ['aaaa', 'bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' ', 2) + ot: ['aaaa', 'bbbb', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('', 2) + ot: ['a', 'a', 'aa bbbb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('b', 2) + ot: ['aaaa ', '', 'bb cccc '] + - cd: r.expr('aaaa bbbb cccc ').split('bb', 2) + ot: ['aaaa ', '', ' cccc '] + - cd: r.expr('aaaa bbbb cccc ').split(' bbbb ', 2) + ot: ['aaaa', 'cccc '] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split('bb', 2) + ot: ['aaaa ', '', ' cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr('aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: ['aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' ').split() + ot: [] + - cd: r.expr(' ').split(null) + ot: [] + - cd: r.expr(' ').split(' ') + ot: ['', '', ''] + - cd: r.expr(' ').split(null, 5) + ot: [] + - cd: r.expr(' ').split(' ', 5) + ot: ['', '', ''] + + - cd: r.expr(' aaaa bbbb cccc ').split() + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr(' aaaa bbbb cccc ').split(null) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr(' aaaa bbbb cccc ').split(' ') + ot: ['', '', 'aaaa', 'bbbb', '', 'cccc', ''] + - cd: r.expr(' aaaa bbbb cccc ').split('b') + ot: [' aaaa ', '', '', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb') + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ') + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb') + ot: [' aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ') + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' aaaa bbbb cccc ').split(null, 3) + ot: ['aaaa', 'bbbb', 'cccc'] + - cd: r.expr(' aaaa bbbb cccc ').split(' ', 5) + ot: ['', '', 'aaaa', 'bbbb', '', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('b', 5) + ot: [' aaaa ', '', '', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb', 3) + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ', 2) + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb', 6) + ot: [' aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 3) + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' aaaa bbbb cccc ').split(null, 2) + ot: ['aaaa', 'bbbb', 'cccc '] + - cd: r.expr("a b ").split(null, 2) + ot: ["a", "b"] + - cd: r.expr(' aaaa bbbb cccc ').split(' ', 4) + ot: ['', '', 'aaaa', 'bbbb', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('b', 4) + ot: [' aaaa ', '', '', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb', 2) + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ', 1) + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb', 5) + ot: [' aaaa ', '', ' cccc b d ', ' e ', '', ' f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 1) + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr(' aaaa bbbb cccc ').split(null, 1) + ot: ['aaaa', 'bbbb cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' ', 2) + ot: ['', '', 'aaaa bbbb cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('b', 2) + ot: [' aaaa ', '', 'bb cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split('bb', 2) + ot: [' aaaa ', '', ' cccc '] + - cd: r.expr(' aaaa bbbb cccc ').split(' bbbb ', 2) + ot: [' aaaa', 'cccc '] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split('bb', 2) + ot: [' aaaa ', '', ' cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e bbbb f'] + - cd: r.expr(' aaaa bbbb cccc b d bb e bbbb f').split(' bbbb ', 2) + ot: [' aaaa', 'cccc b d bb e', 'f'] + + - cd: r.expr("abc-dEf-GHJ").upcase() + ot: "ABC-DEF-GHJ" + - cd: r.expr("abc-dEf-GHJ").downcase() + ot: "abc-def-ghj" + + # Same 3.0-3.2 caveats + - py: + cd: r.expr(u"f\u00e9oo").split("") + ot: [u"f", u"\u00e9", u"o", u"o"] + py3.0: r.expr("f\u00e9oo").split("") + py3.1: r.expr("f\u00e9oo").split("") + py3.2: r.expr("f\u00e9oo").split("") + cd: r.expr("f\u00e9oo").split("") + ot: ["f", "\u00e9", "o", "o"] + + - py: + cd: r.expr(u"fe\u0301oo").split("") + ot: [u"f", u"e\u0301", u"o", u"o"] + py3.0: r.expr("fe\u0301oo").split("") + py3.1: r.expr("fe\u0301oo").split("") + py3.2: r.expr("fe\u0301oo").split("") + cd: r.expr("fe\u0301oo").split("") + ot: ["f", "e\u0301", "o", "o"] + + ## Unicode spacing characters. + + ## original set from previous work: + - cd: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + py: + cd: r.expr(u"foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + ot: ["foo", "bar", "baz", "quux", "fred", "barney", "wilma"] + py3.0: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + py3.1: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + py3.2: r.expr("foo bar\tbaz\nquux\rfred\u000bbarney\u000cwilma").split() + ot: ["foo", "bar", "baz", "quux", "fred", "barney", "wilma"] + + ## some specialized Unicode horrors: + ## - U+00A0 is nonbreaking space and is in the Zs category + ## - U+0085 is the next line character and is not in the Zs category but is considered whitespace + ## - U+2001 is em quad space and is in the Zs category + ## - U+200B is a zero width space and is not in the Zs category and is not considered whitespace + ## - U+2060 is a word joining zero width nonbreaking space and is NOT in any of the Z categories + ## - U+2028 is a line separator and is in the Zl category + ## - U+2029 is a paragraph separator and is in the Zp category + - py: + cd: r.expr(u"foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + ot: ["foo", "bar", u"baz\u2060quux", "fred", "barney", "wilma", u"betty\u200b"] + py3.0: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + py3.1: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + py3.2: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + cd: r.expr("foo\u00a0bar\u2001baz\u2060quux\u2028fred\u2028barney\u2029wilma\u0085betty\u200b").split() + ot: ["foo", "bar", "baz\u2060quux", "fred", "barney", "wilma", "betty\u200b"] diff --git a/ext/librethinkdbxx/test/upstream/datum/typeof.yaml b/ext/librethinkdbxx/test/upstream/datum/typeof.yaml new file mode 100644 index 00000000..1f03f53e --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/typeof.yaml @@ -0,0 +1,14 @@ +desc: These tests test the type of command +tests: + + # Method form + - cd: r.expr(null).type_of() + ot: 'NULL' + + # Prefix form + - cd: r.type_of(null) + ot: 'NULL' + + # Error cases + - js: r(null).typeOf(1) + ot: err('ReqlCompileError', 'Expected 1 argument but found 2.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/datum/uuid.yaml b/ext/librethinkdbxx/test/upstream/datum/uuid.yaml new file mode 100644 index 00000000..003e8604 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/datum/uuid.yaml @@ -0,0 +1,20 @@ +desc: Test that UUIDs work +tests: + - cd: r.uuid() + ot: uuid() + - cd: r.expr(r.uuid()) + ot: uuid() + - cd: r.type_of(r.uuid()) + ot: 'STRING' + - cd: r.uuid().ne(r.uuid()) + ot: true + - cd: r.uuid('magic') + ot: ('97dd10a5-4fc4-554f-86c5-0d2c2e3d5330') + - cd: r.uuid('magic').eq(r.uuid('magic')) + ot: true + - cd: r.uuid('magic').ne(r.uuid('beans')) + ot: true + - py: r.expr([1,2,3,4,5,6,7,8,9,10]).map(lambda u:r.uuid()).distinct().count() + js: r([1,2,3,4,5,6,7,8,9,10]).map(function(u) {return r.uuid();}).distinct().count() + rb: r.expr([1,2,3,4,5,6,7,8,9,10]).map {|u| r.uuid()}.distinct().count() + ot: 10 diff --git a/ext/librethinkdbxx/test/upstream/default.yaml b/ext/librethinkdbxx/test/upstream/default.yaml new file mode 100644 index 00000000..e7ba44c0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/default.yaml @@ -0,0 +1,270 @@ +desc: Tests r.default +tests: + - cd: r.expr(1).default(2) + ot: 1 + - cd: r.expr(null).default(2) + ot: 2 + - cd: r.expr({})['b'].default(2) + js: r.expr({})('b').default(2) + ot: 2 + - cd: r.expr(r.expr('a')['b']).default(2) + js: r.expr(r.expr('a')('b')).default(2) + ot: err("ReqlQueryLogicError", "Cannot perform bracket on a non-object non-sequence `\"a\"`.", []) + - rb: r.expr([]).reduce{|a,b| a+b}.default(2) + py: r.expr([]).reduce(lambda a,b:a+b).default(2) + js: r.expr([]).reduce(function(a,b){return a+b}).default(2) + ot: 2 + - rb: r.expr([]).union([]).reduce{|a,b| a+b}.default(2) + py: r.expr([]).union([]).reduce(lambda a,b:a+b).default(2) + js: r.expr([]).union([]).reduce(function(a,b){return a+b}).default(2) + ot: 2 + - rb: r.expr('a').reduce{|a,b| a+b}.default(2) + py: r.expr('a').reduce(lambda a,b:a+b).default(2) + js: r.expr('a').reduce(function(a,b){return a+b}).default(2) + ot: err("ReqlQueryLogicError", "Cannot convert STRING to SEQUENCE", []) + - cd: (r.expr(null) + 5).default(2) + js: (r.expr(null).add(5)).default(2) + ot: 2 + - cd: (5 + r.expr(null)).default(2) + js: (r.expr(5).add(null)).default(2) + ot: 2 + - cd: (5 - r.expr(null)).default(2) + js: (r.expr(5).sub(null)).default(2) + ot: 2 + - cd: (r.expr(null) - 5).default(2) + js: (r.expr(null).sub(5)).default(2) + ot: 2 + - cd: (r.expr('a') + 5).default(2) + js: (r.expr('a').add(5)).default(2) + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", []) + - cd: (5 + r.expr('a')).default(2) + js: (r.expr(5).add('a')).default(2) + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + - cd: (r.expr('a') - 5).default(2) + js: (r.expr('a').sub(5)).default(2) + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + - cd: (5 - r.expr('a')).default(2) + js: (r.expr(5).sub('a')).default(2) + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + - cd: r.expr(1).default(r.error()) + ot: 1 + - cd: r.expr(null).default(r.error()) + ot: (null) + - cd: r.expr({})['b'].default(r.error()) + js: r.expr({})('b').default(r.error()) + ot: err("ReqlNonExistenceError", "No attribute `b` in object:", []) + - rb: r.expr([]).reduce{|a,b| a+b}.default(r.error) + py: r.expr([]).reduce(lambda a,b:a+b).default(r.error) + js: r.expr([]).reduce(function(a,b){return a+b}).default(r.error) + ot: err("ReqlNonExistenceError", "Cannot reduce over an empty stream.", []) + - rb: r.expr([]).union([]).reduce{|a,b| a+b}.default(r.error) + py: r.expr([]).union([]).reduce(lambda a,b:a+b).default(r.error) + js: r.expr([]).union([]).reduce(function(a,b){return a+b}).default(r.error) + ot: err("ReqlNonExistenceError", "Cannot reduce over an empty stream.", []) + - cd: (r.expr(null) + 5).default(r.error) + js: (r.expr(null).add(5)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + - cd: (5 + r.expr(null)).default(r.error) + js: (r.expr(5).add(null)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + - cd: (5 - r.expr(null)).default(r.error) + js: (r.expr(5).sub(null)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + - cd: (r.expr(null) - 5).default(r.error) + js: (r.expr(null).sub(5)).default(r.error) + ot: err("ReqlNonExistenceError", "Expected type NUMBER but found NULL.", []) + + - rb: r.expr(1).default{|e| e} + py: r.expr(1).default(lambda e:e) + js: r.expr(1).default(function(e){return e}) + ot: 1 + - cd: r.expr(null).default{|e| e} + py: r.expr(null).default(lambda e:e) + js: r.expr(null).default(function(e){return e}) + ot: (null) + - cd: r.expr({})['b'].default{|e| e} + py: r.expr({})['b'].default(lambda e:e) + js: r.expr({})('b').default(function(e){return e}) + ot: "No attribute `b` in object:\n{}" + - cd: r.expr([]).reduce{|a,b| a+b}.default{|e| e} + py: r.expr([]).reduce(lambda a,b:a+b).default(lambda e:e) + js: r.expr([]).reduce(function(a,b){return a+b}).default(function(e){return e}) + ot: ("Cannot reduce over an empty stream.") + - cd: r.expr([]).union([]).reduce{|a,b| a+b}.default{|e| e} + py: r.expr([]).union([]).reduce(lambda a,b:a+b).default(lambda e:e) + js: r.expr([]).union([]).reduce(function(a,b){return a+b}).default(function(e){return e}) + ot: ("Cannot reduce over an empty stream.") + - cd: (r.expr(null) + 5).default{|e| e} + py: (r.expr(null) + 5).default(lambda e:e) + js: (r.expr(null).add(5)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + - cd: (5 + r.expr(null)).default{|e| e} + py: (5 + r.expr(null)).default(lambda e:e) + js: (r.expr(5).add(null)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + - cd: (5 - r.expr(null)).default{|e| e} + py: (5 - r.expr(null)).default(lambda e:e) + js: (r.expr(5).sub(null)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + - cd: (r.expr(null) - 5).default{|e| e} + py: (r.expr(null) - 5).default(lambda e:e) + js: (r.expr(null).sub(5)).default(function(e){return e}) + ot: ("Expected type NUMBER but found NULL.") + + - def: arr = r.expr([{'a':1},{'a':null},{}]).order_by('a') + + - cd: arr.filter{|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1)) + js: arr.filter(function(x){return x('a').eq(1)}) + ot: [{'a':1}] + - cd: arr.filter(:default => false){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=False) + js: arr.filter(function(x){return x('a').eq(1)}, {'default':false}) + ot: [{'a':1}] + - cd: arr.filter(:default => true){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=True) + js: arr.filter(function(x){return x('a').eq(1)}, {'default':true}) + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + - cd: arr.filter(:default => r.js('true')){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=r.js('true')) + js: arr.filter(function(x) { return x('a').eq(1) }, { 'default':r.js('true') }) + ot: [{}, {'a':1}] + - cd: arr.filter(:default => r.js('false')){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=r.js('false')) + js: arr.filter(function(x) { return x('a').eq(1) }, { 'default':r.js('false') }) + ot: [{'a':1}] + - cd: arr.filter(:default => r.error){|x| x['a'].eq(1)} + py: arr.filter(lambda x:x['a'].eq(1), default=r.error()) + js: arr.filter(function(x){return x('a').eq(1)}, {'default':r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.expr(false).do{|d| arr.filter(:default => d){|x| x['a'].eq(1)}} + py: r.expr(False).do(lambda d:arr.filter(lambda x:x['a'].eq(1), default=d)) + js: r.expr(false).do(function(d){return arr.filter(function(x){return x('a').eq(1)}, {default:d})}) + ot: [{'a':1}] + - cd: r.expr(true).do{|d| arr.filter(:default => d){|x| x['a'].eq(1)}}.orderby('a') + py: r.expr(True).do(lambda d:arr.filter(lambda x:x['a'].eq(1), default=d)).order_by('a') + js: r.expr(true).do(function(d){return arr.filter(function(x){return x('a').eq(1)}, {default:d})}).orderBy('a') + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + + - cd: arr.filter{|x| x['a'].default(0).eq(1)} + py: arr.filter(lambda x:x['a'].default(0).eq(1)) + js: arr.filter(function(x){return x('a').default(0).eq(1)}) + ot: [{'a':1}] + - cd: arr.filter{|x| x['a'].default(1).eq(1)}.orderby('a') + py: arr.filter(lambda x:x['a'].default(1).eq(1)).order_by('a') + js: arr.filter(function(x){return x('a').default(1).eq(1)}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: arr.filter{|x| x['a'].default(r.error).eq(1)} + py: arr.filter(lambda x:x['a'].default(r.error()).eq(1)) + js: arr.filter(function(x){return x('a').default(r.error()).eq(1)}) + ot: [{'a':1}] + # gets caught by `filter` default + + - cd: r.expr(0).do{|i| arr.filter{|x| x['a'].default(i).eq(1)}} + py: r.expr(0).do(lambda i:arr.filter(lambda x:x['a'].default(i).eq(1))) + js: r.expr(0).do(function(i){return arr.filter(function(x){return x('a').default(i).eq(1)})}) + ot: [{'a':1}] + - cd: r.expr(1).do{|i| arr.filter{|x| x['a'].default(i).eq(1)}}.orderby('a') + py: r.expr(1).do(lambda i:arr.filter(lambda x:x['a'].default(i).eq(1))).order_by('a') + js: r.expr(1).do(function(i){return arr.filter(function(x){return x('a').default(i).eq(1)})}).orderBy('a') + ot: ([{},{'a':null},{'a':1}]) + + - cd: arr.filter{|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2))) + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}) + ot: [{'a':1}] + - cd: arr.filter(:default => false){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=False) + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:false}) + ot: [{'a':1}] + - cd: arr.filter(:default => true){|x| x['a'].eq(1).or(x['a']['b'].eq(2))}.orderby('a') + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=True).order_by('a') + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:true}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: arr.filter(:default => r.error){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: arr.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=r.error()) + js: arr.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.table_create('default_test') + ot: partial({'tables_created':1}) + + - cd: r.table('default_test').insert(arr) + ot: ({'deleted':0,'replaced':0,'generated_keys':arrlen(3,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':3}) + + - def: tbl = r.table('default_test').order_by('a').pluck('a') + + - cd: tbl.filter{|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1)) + js: tbl.filter(function(x){return x('a').eq(1)}) + ot: [{'a':1}] + - cd: tbl.filter(:default => false){|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1), default=False) + js: tbl.filter(function(x){return x('a').eq(1)}, {'default':false}) + ot: [{'a':1}] + - cd: tbl.filter(:default => true){|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1), default=True) + js: tbl.filter(function(x){return x('a').eq(1)}, {'default':true}) + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + - cd: tbl.filter(:default => r.error){|x| x['a'].eq(1)} + py: tbl.filter(lambda x:x['a'].eq(1), default=r.error()) + js: tbl.filter(function(x){return x('a').eq(1)}, {'default':r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.expr(false).do{|d| tbl.filter(:default => d){|x| x['a'].eq(1)}} + py: r.expr(False).do(lambda d:tbl.filter(lambda x:x['a'].eq(1), default=d)) + js: r.expr(false).do(function(d){return tbl.filter(function(x){return x('a').eq(1)}, {default:d})}) + ot: [{'a':1}] + - cd: r.expr(true).do{|d| tbl.filter(:default => d){|x| x['a'].eq(1)}}.orderby('a') + py: r.expr(True).do(lambda d:tbl.filter(lambda x:x['a'].eq(1), default=d)).order_by('a') + js: r.expr(true).do(function(d){return tbl.filter(function(x){return x('a').eq(1)}, {default:d})}).orderBy('a') + ot: [{}, {'a':1}] + # `null` compares not equal to 1 with no error + + - cd: tbl.filter{|x| x['a'].default(0).eq(1)} + py: tbl.filter(lambda x:x['a'].default(0).eq(1)) + js: tbl.filter(function(x){return x('a').default(0).eq(1)}) + ot: [{'a':1}] + - cd: tbl.filter{|x| x['a'].default(1).eq(1)}.orderby('a') + py: tbl.filter(lambda x:x['a'].default(1).eq(1)).order_by('a') + js: tbl.filter(function(x){return x('a').default(1).eq(1)}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: tbl.filter{|x| x['a'].default(r.error).eq(1)} + py: tbl.filter(lambda x:x['a'].default(r.error()).eq(1)) + js: tbl.filter(function(x){return x('a').default(r.error()).eq(1)}) + ot: [{'a':1}] + # gets caught by `filter` default + + - cd: r.expr(0).do{|i| tbl.filter{|x| x['a'].default(i).eq(1)}} + py: r.expr(0).do(lambda i:tbl.filter(lambda x:x['a'].default(i).eq(1))) + js: r.expr(0).do(function(i){return tbl.filter(function(x){return x('a').default(i).eq(1)})}) + ot: [{'a':1}] + - cd: r.expr(1).do{|i| tbl.filter{|x| x['a'].default(i).eq(1)}}.orderby('a') + py: r.expr(1).do(lambda i:tbl.filter(lambda x:x['a'].default(i).eq(1))).order_by('a') + js: r.expr(1).do(function(i){return tbl.filter(function(x){return x('a').default(i).eq(1)})}).orderBy('a') + ot: ([{},{'a':null},{'a':1}]) + + - cd: tbl.filter{|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2))) + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}) + ot: [{'a':1}] + - cd: tbl.filter(:default => false){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=False) + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:false}) + ot: [{'a':1}] + - cd: tbl.filter(:default => true){|x| x['a'].eq(1).or(x['a']['b'].eq(2))}.orderby('a') + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=True).order_by('a') + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:true}).orderBy('a') + ot: ([{}, {'a':null}, {'a':1}]) + - cd: tbl.filter(:default => r.error){|x| x['a'].eq(1).or(x['a']['b'].eq(2))} + py: tbl.filter(lambda x:r.or_(x['a'].eq(1), x['a']['b'].eq(2)), default=r.error()) + js: tbl.filter(function(x){return x('a').eq(1).or(x('a')('b').eq(2))}, {default:r.error()}) + ot: err("ReqlNonExistenceError", "No attribute `a` in object:", []) + + - cd: r.table_drop('default_test') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/geo/constructors.yaml b/ext/librethinkdbxx/test/upstream/geo/constructors.yaml new file mode 100644 index 00000000..9991c582 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/constructors.yaml @@ -0,0 +1,64 @@ +desc: Test geo constructors +tests: + # Point + - cd: r.point(0, 0) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 0], 'type':'Point'}) + - cd: r.point(0, -90) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, -90], 'type':'Point'}) + - cd: r.point(0, 90) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 90], 'type':'Point'}) + - cd: r.point(-180, 0) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[-180, 0], 'type':'Point'}) + - cd: r.point(180, 0) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[180, 0], 'type':'Point'}) + - cd: r.point(0, -91) + ot: err('ReqlQueryLogicError', 'Latitude must be between -90 and 90. Got -91.', [0]) + - cd: r.point(0, 91) + ot: err('ReqlQueryLogicError', 'Latitude must be between -90 and 90. Got 91.', [0]) + - cd: r.point(-181, 0) + ot: err('ReqlQueryLogicError', 'Longitude must be between -180 and 180. Got -181.', [0]) + - cd: r.point(181, 0) + ot: err('ReqlQueryLogicError', 'Longitude must be between -180 and 180. Got 181.', [0]) + + # Line + - cd: r.line() + ot: err('ReqlCompileError', 'Expected 2 or more arguments but found 0.', [0]) + - cd: r.line([0,0]) + ot: err('ReqlCompileError', 'Expected 2 or more arguments but found 1.', [0]) + - cd: r.line([0,0], [0,0]) + ot: err('ReqlQueryLogicError', 'Invalid LineString. Are there antipodal or duplicate vertices?', [0]) + - cd: r.line([0,0], [0,1]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1]], 'type':'LineString'}) + - cd: r.line([0,0], [1]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 1 element array instead of a 2 element one.', [0]) + - cd: r.line([0,0], [1,0,0]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 3 element array instead of a 2 element one.', [0]) + - cd: r.line([0,0], [0,1], [0,0]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1], [0,0]], 'type':'LineString'}) + - cd: r.line(r.point(0,0), r.point(0,1), r.point(0,0)) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1], [0,0]], 'type':'LineString'}) + - cd: r.line(r.point(0,0), r.point(1,0), r.line([0,0], [1,0])) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Point` but found `LineString`.', [0]) + + # Polygon + - cd: r.polygon() + ot: err('ReqlCompileError', 'Expected 3 or more arguments but found 0.', [0]) + - cd: r.polygon([0,0]) + ot: err('ReqlCompileError', 'Expected 3 or more arguments but found 1.', [0]) + - cd: r.polygon([0,0], [0,0]) + ot: err('ReqlCompileError', 'Expected 3 or more arguments but found 2.', [0]) + - cd: r.polygon([0,0], [0,0], [0,0], [0,0]) + ot: err('ReqlQueryLogicError', 'Invalid LinearRing. Are there antipodal or duplicate vertices? Is it self-intersecting?', [0]) + - cd: r.polygon([0,0], [0,1], [1,0]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [0,1], [1,0], [0,0]) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [0,1], [1,0], [-1,0.5]) + ot: err('ReqlQueryLogicError', 'Invalid LinearRing. Are there antipodal or duplicate vertices? Is it self-intersecting?', [0]) + - cd: r.polygon([0,0], [0,1], [0]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 1 element array instead of a 2 element one.', [0]) + - cd: r.polygon([0,0], [0,1], [0,1,0]) + ot: err('ReqlQueryLogicError', 'Expected point coordinate pair. Got 3 element array instead of a 2 element one.', [0]) + - cd: r.polygon(r.point(0,0), r.point(0,1), r.line([0,0], [0,1])) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Point` but found `LineString`.', [0]) + diff --git a/ext/librethinkdbxx/test/upstream/geo/geojson.yaml b/ext/librethinkdbxx/test/upstream/geo/geojson.yaml new file mode 100644 index 00000000..cc0be877 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/geojson.yaml @@ -0,0 +1,31 @@ +desc: Test geoJSON conversion +tests: + # Basic conversion + - cd: r.geojson({'coordinates':[0, 0], 'type':'Point'}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 0], 'type':'Point'}) + - cd: r.geojson({'coordinates':[[0,0], [0,1]], 'type':'LineString'}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0,0], [0,1]], 'type':'LineString'}) + - cd: r.geojson({'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0], [0,1], [1,0], [0,0]]], 'type':'Polygon'}) + + # Wrong / missing fields + - cd: r.geojson({'coordinates':[[], 0], 'type':'Point'}) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found ARRAY.', [0]) + - cd: r.geojson({'coordinates':true, 'type':'Point'}) + ot: err('ReqlQueryLogicError', 'Expected type ARRAY but found BOOL.', [0]) + - cd: r.geojson({'type':'Point'}) + ot: err('ReqlNonExistenceError', 'No attribute `coordinates` in object:', [0]) + - cd: r.geojson({'coordinates':[0, 0]}) + ot: err('ReqlNonExistenceError', 'No attribute `type` in object:', [0]) + - cd: r.geojson({'coordinates':[0, 0], 'type':'foo'}) + ot: err('ReqlQueryLogicError', 'Unrecognized GeoJSON type `foo`.', [0]) + - cd: r.geojson({'coordinates':[0, 0], 'type':'Point', 'foo':'wrong'}) + ot: err('ReqlQueryLogicError', 'Unrecognized field `foo` found in geometry object.', [0]) + + # Unsupported features + - cd: r.geojson({'coordinates':[0, 0], 'type':'Point', 'crs':null}) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[0, 0], 'type':'Point', 'crs':null}) + - js: r.geojson({'coordinates':[0, 0], 'type':'Point', 'crs':{'type':'name', 'properties':{'name':'test'}}}) + ot: err('ReqlQueryLogicError', 'Non-default coordinate reference systems are not supported in GeoJSON objects. Make sure the `crs` field of the geometry is null or non-existent.', [0]) + - cd: r.geojson({'coordinates':[0, 0], 'type':'MultiPoint'}) + ot: err('ReqlQueryLogicError', 'GeoJSON type `MultiPoint` is not supported.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/geo/indexing.yaml b/ext/librethinkdbxx/test/upstream/geo/indexing.yaml new file mode 100644 index 00000000..059f2929 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/indexing.yaml @@ -0,0 +1,208 @@ +desc: Test ReQL interface to geo indexes +table_variable_name: tbl +tests: + - def: rows = [{'id':0, 'g':r.point(10,10), 'm':[r.point(0,0),r.point(1,0),r.point(2,0)]}, + {'id':1, 'g':r.polygon([0,0], [0,1], [1,1], [1,0])}, + {'id':2, 'g':r.line([0.000002,-1], [-0.000001,1])}] + + - cd: tbl.insert(rows) + ot: ({'deleted':0,'inserted':3,'skipped':0,'errors':0,'replaced':0,'unchanged':0}) + + - rb: tbl.index_create('g', :geo=>true) + py: tbl.index_create('g', geo=true) + js: tbl.indexCreate('g', {'geo':true}) + ot: {'created':1} + - rb: tbl.index_create('m', :geo=>true, :multi=>true) + py: tbl.index_create('m', geo=true, multi=true) + js: tbl.indexCreate('m', {'geo':true, 'multi':true}) + ot: {'created':1} + - cd: tbl.index_create('other') + ot: {'created':1} + # r.point is deterministic and can be used in an index function + - rb: tbl.index_create('point_det'){ |x| r.point(x, x) } + py: tbl.index_create('point_det', lambda x: r.point(x, x) ) + js: tbl.indexCreate('point_det', function(x) {return r.point(x, x);} ) + ot: {'created':1} + + - cd: tbl.index_wait() + + # r.line (and friends) are non-deterministic across servers and should be disallowed + # in index functions + - rb: tbl.index_create('point_det'){ |x| r.line(x, x) } + py: tbl.index_create('point_det', lambda x: r.line(x, x) ) + js: tbl.indexCreate('point_det', function(x) {return r.line(x, x);} ) + ot: err('ReqlQueryLogicError', 'Could not prove function deterministic. Index functions must be deterministic.') + + - js: tbl.get_intersecting(r.point(0,0), {'index':'other'}).count() + py: tbl.get_intersecting(r.point(0,0), index='other').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'other').count() + ot: err('ReqlQueryLogicError', 'Index `other` is not a geospatial index. get_intersecting can only be used with a geospatial index.', [0]) + - js: tbl.get_intersecting(r.point(0,0), {'index':'missing'}).count() + py: tbl.get_intersecting(r.point(0,0), index='missing').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'missing').count() + ot: err_regex('ReqlOpFailedError', 'Index `missing` was not found on table `[a-zA-Z0-9_]+.[a-zA-Z0-9_]+`[.]', [0]) + - cd: tbl.get_intersecting(r.point(0,0)).count() + ot: err('ReqlQueryLogicError', 'get_intersecting requires an index argument.', [0]) + - js: tbl.get_all(0, {'index':'g'}).count() + py: tbl.get_all(0, index='g').count() + rb: tbl.get_all(0, :index=>'g').count() + ot: err('ReqlQueryLogicError', 'Index `g` is a geospatial index. Only get_nearest and get_intersecting can use a geospatial index.', [0]) + - js: tbl.between(0, 1, {'index':'g'}).count() + py: tbl.between(0, 1, index='g').count() + rb: tbl.between(0, 1, :index=>'g').count() + ot: err('ReqlQueryLogicError', 'Index `g` is a geospatial index. Only get_nearest and get_intersecting can use a geospatial index.', [0]) + - js: tbl.order_by({'index':'g'}).count() + py: tbl.order_by(index='g').count() + rb: tbl.order_by(:index=>'g').count() + ot: err('ReqlQueryLogicError', 'Index `g` is a geospatial index. Only get_nearest and get_intersecting can use a geospatial index.', [0]) + - js: tbl.between(0, 1).get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.between(0, 1).get_intersecting(r.point(0,0), index='g').count() + rb: tbl.between(0, 1).get_intersecting(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'Between' object has no attribute 'get_intersecting'") + - js: tbl.get_all(0).get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.get_all(0).get_intersecting(r.point(0,0), index='g').count() + rb: tbl.get_all(0).get_intersecting(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found SELECTION:', [0]) + py: err('AttributeError', "'GetAll' object has no attribute 'get_intersecting'") + - js: tbl.order_by({'index':'id'}).get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.order_by(index='id').get_intersecting(r.point(0,0), index='g').count() + rb: tbl.order_by(:index=>'id').get_intersecting(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'OrderBy' object has no attribute 'get_intersecting'") + - js: tbl.get_intersecting(r.point(0,0), {'index':'id'}).count() + py: tbl.get_intersecting(r.point(0,0), index='id').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'id').count() + ot: err('ReqlQueryLogicError', 'get_intersecting cannot use the primary index.', [0]) + + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(0,0), index='g').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').count() + ot: 1 + - js: tbl.get_intersecting(r.point(10,10), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(10,10), index='g').count() + rb: tbl.get_intersecting(r.point(10,10), :index=>'g').count() + ot: 1 + - js: tbl.get_intersecting(r.point(0.5,0.5), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(0.5,0.5), index='g').count() + rb: tbl.get_intersecting(r.point(0.5,0.5), :index=>'g').count() + ot: 1 + - js: tbl.get_intersecting(r.point(20,20), {'index':'g'}).count() + py: tbl.get_intersecting(r.point(20,20), index='g').count() + rb: tbl.get_intersecting(r.point(20,20), :index=>'g').count() + ot: 0 + - js: tbl.get_intersecting(r.polygon([0,0], [1,0], [1,1], [0,1]), {'index':'g'}).count() + py: tbl.get_intersecting(r.polygon([0,0], [1,0], [1,1], [0,1]), index='g').count() + rb: tbl.get_intersecting(r.polygon([0,0], [1,0], [1,1], [0,1]), :index=>'g').count() + ot: 2 + - js: tbl.get_intersecting(r.line([0,0], [10,10]), {'index':'g'}).count() + py: tbl.get_intersecting(r.line([0,0], [10,10]), index='g').count() + rb: tbl.get_intersecting(r.line([0,0], [10,10]), :index=>'g').count() + ot: 3 + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).type_of() + py: tbl.get_intersecting(r.point(0,0), index='g').type_of() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').type_of() + ot: ("SELECTION") + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).filter(true).type_of() + py: tbl.get_intersecting(r.point(0,0), index='g').filter(true).type_of() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').filter(true).type_of() + ot: ("SELECTION") + - js: tbl.get_intersecting(r.point(0,0), {'index':'g'}).map(r.row).type_of() + py: tbl.get_intersecting(r.point(0,0), index='g').map(r.row).type_of() + rb: tbl.get_intersecting(r.point(0,0), :index=>'g').map{|x|x}.type_of() + ot: ("STREAM") + + - js: tbl.get_intersecting(r.point(0,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(0,0), index='m').count() + rb: tbl.get_intersecting(r.point(0,0), :index=>'m').count() + ot: 1 + - js: tbl.get_intersecting(r.point(1,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(1,0), index='m').count() + rb: tbl.get_intersecting(r.point(1,0), :index=>'m').count() + ot: 1 + - js: tbl.get_intersecting(r.point(2,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(2,0), index='m').count() + rb: tbl.get_intersecting(r.point(2,0), :index=>'m').count() + ot: 1 + - js: tbl.get_intersecting(r.point(3,0), {'index':'m'}).count() + py: tbl.get_intersecting(r.point(3,0), index='m').count() + rb: tbl.get_intersecting(r.point(3,0), :index=>'m').count() + ot: 0 + # The document is emitted once for each match. + - js: tbl.get_intersecting(r.polygon([0,0], [0,1], [1,1], [1,0]), {'index':'m'}).count() + py: tbl.get_intersecting(r.polygon([0,0], [0,1], [1,1], [1,0]), index='m').count() + rb: tbl.get_intersecting(r.polygon([0,0], [0,1], [1,1], [1,0]), :index=>'m').count() + ot: 2 + + + - js: tbl.get_nearest(r.point(0,0), {'index':'other'}) + py: tbl.get_nearest(r.point(0,0), index='other') + rb: tbl.get_nearest(r.point(0,0), :index=>'other') + ot: err('ReqlQueryLogicError', 'Index `other` is not a geospatial index. get_nearest can only be used with a geospatial index.', [0]) + - js: tbl.get_nearest(r.point(0,0), {'index':'missing'}) + py: tbl.get_nearest(r.point(0,0), index='missing') + rb: tbl.get_nearest(r.point(0,0), :index=>'missing') + ot: err_regex('ReqlOpFailedError', 'Index `missing` was not found on table `[a-zA-Z0-9_]+.[a-zA-Z0-9_]+`[.]', [0]) + - cd: tbl.get_nearest(r.point(0,0)) + ot: err('ReqlQueryLogicError', 'get_nearest requires an index argument.', [0]) + - js: tbl.between(0, 1).get_nearest(r.point(0,0), {'index':'g'}).count() + py: tbl.between(0, 1).get_nearest(r.point(0,0), index='g').count() + rb: tbl.between(0, 1).get_nearest(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'Between' object has no attribute 'get_nearest'") + - js: tbl.get_all(0).get_nearest(r.point(0,0), {'index':'g'}).count() + py: tbl.get_all(0).get_nearest(r.point(0,0), index='g').count() + rb: tbl.get_all(0).get_nearest(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found SELECTION:', [0]) + py: err('AttributeError', "'GetAll' object has no attribute 'get_nearest'") + - js: tbl.order_by({'index':'id'}).get_nearest(r.point(0,0), {'index':'g'}).count() + py: tbl.order_by(index='id').get_nearest(r.point(0,0), index='g').count() + rb: tbl.order_by(:index=>'id').get_nearest(r.point(0,0), :index=>'g').count() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [0]) + py: err('AttributeError', "'OrderBy' object has no attribute 'get_nearest'") + - js: tbl.get_nearest(r.point(0,0), {'index':'id'}).count() + py: tbl.get_nearest(r.point(0,0), index='id').count() + rb: tbl.get_nearest(r.point(0,0), :index=>'id').count() + ot: err('ReqlQueryLogicError', 'get_nearest cannot use the primary index.', [0]) + + - js: tbl.get_nearest(r.point(0,0), {'index':'g'}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.055659745396754216,'doc':{'id':2}}]) + - js: tbl.get_nearest(r.point(-0.000001,1), {'index':'g'}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(-0.000001,1), index='g').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(-0.000001,1), :index=>'g').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':2}},{'dist':0.11130264976984369,'doc':{'id':1}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':1565110}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1565110).pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1565110).pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.055659745396754216,'doc':{'id':2}},{'dist':1565109.0992178896,'doc':{'id':0}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':1565110, 'max_results':2}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1565110, max_results=2).pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1565110, :max_results=>2).pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.055659745396754216,'doc':{'id':2}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':10000000}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=10000000).pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>10000000).pluck('dist', {'doc':'id'}) + ot: err('ReqlQueryLogicError', 'The distance has become too large for continuing the indexed nearest traversal. Consider specifying a smaller `max_dist` parameter. (Radius must be smaller than a quarter of the circumference along the minor axis of the reference ellipsoid. Got 10968937.995244588703m, but must be smaller than 9985163.1855612862855m.)', [0]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g', 'max_dist':1566, 'unit':'km'}).pluck('dist', {'doc':'id'}) + py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1566, unit='km').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1566, :unit=>'km').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0,'doc':{'id':1}},{'dist':0.00005565974539675422,'doc':{'id':2}},{'dist':1565.1090992178897,'doc':{'id':0}}]) + - py: tbl.get_nearest(r.point(0,0), index='g', max_dist=1, geo_system='unit_sphere').pluck('dist', {'doc':'id'}) + rb: tbl.get_nearest(r.point(0,0), :index=>'g', :max_dist=>1, :geo_system=>'unit_sphere').pluck('dist', {'doc':'id'}) + ot: ([{'dist':0, 'doc':{'id':1}}, {'dist':8.726646259990191e-09, 'doc':{'id':2}}, {'dist':0.24619691677893205, 'doc':{'id':0}}]) + - js: tbl.get_nearest(r.point(0,0), {'index':'g'}).type_of() + py: tbl.get_nearest(r.point(0,0), index='g').type_of() + rb: tbl.get_nearest(r.point(0,0), :index=>'g').type_of() + ot: ("ARRAY") + - js: tbl.get_nearest(r.point(0,0), {'index':'g'}).map(r.row).type_of() + py: tbl.get_nearest(r.point(0,0), index='g').map(r.row).type_of() + rb: tbl.get_nearest(r.point(0,0), :index=>'g').map{|x|x}.type_of() + ot: ("ARRAY") diff --git a/ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml b/ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml new file mode 100644 index 00000000..18a643a0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/intersection_inclusion.yaml @@ -0,0 +1,119 @@ +desc: Test intersects and includes semantics +tests: + # Intersects + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(1.5,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(2.5,2.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.5,1.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.05,1.05)) + ot: true + # Our current semantics: we define polygons as closed, so points that are exactly *on* the outline of a polygon do intersect + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(2,2)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.point(2,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([1.5,1.5], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([1.5,1.5], [2,1.5])) + ot: true + # (...with holes in the polygon being closed with respect to the polygon, i.e. the set cut out is open) + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.1,1.1)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.point(1.5,1.1)) + ot: true + # ... lines are interpreted as closed sets as well, so even if they meet only at their end points, we consider them as intersecting. + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([2,2], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([2,1.5], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.line([1.5,1.5], [3,3])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([1.5,1.5], [2.5,1.5], [2.5,2.5], [1.5,2.5])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).intersects(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])) + ot: false + # Polygons behave like lines in that respect + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([2,1.1], [3,1.1], [3,1.9], [2,1.9])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).intersects(r.polygon([2,2], [3,2], [3,3], [2,3])) + ot: false + - cd: r.point(1,1).intersects(r.point(1.5,1.5)) + ot: false + - cd: r.point(1,1).intersects(r.point(1,1)) + ot: true + - cd: r.line([1,1], [2,1]).intersects(r.point(1,1)) + ot: true + # This one currently fails due to numeric precision problems. + #- cd: r.line([1,0], [2,0]).intersects(r.point(1.5,0)) + # ot: true + - cd: r.line([1,1], [1,2]).intersects(r.point(1,1.8)) + ot: true + - cd: r.line([1,0], [2,0]).intersects(r.point(1.8,0)) + ot: true + - cd: r.line([1,1], [2,1]).intersects(r.point(1.5,1.5)) + ot: false + - cd: r.line([1,1], [2,1]).intersects(r.line([2,1], [3,1])) + ot: true + # intersects on an array/stream + - cd: r.expr([r.point(1, 0), r.point(3,0), r.point(2, 0)]).intersects(r.line([0,0], [2, 0])).count() + ot: 2 + + # Includes + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(1.5,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(2.5,2.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.5,1.5)) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.05,1.05)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(2,2)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.point(2,1.5)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([1.5,1.5], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([1.5,1.5], [2,1.5])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.1,1.1)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.point(1.5,1.1)) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([2,2], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([2,1.5], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([2,1], [2,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.line([1.5,1.5], [3,3])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1,1], [2,1], [2,2], [1,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1.5,1.5], [2,1.5], [2,2], [1.5,2])) + ot: true + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([1.5,1.5], [2.5,1.5], [2.5,2.5], [1.5,2.5])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.polygon([1.2,1.2], [1.8,1.2], [1.8,1.8], [1.2,1.8])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).polygon_sub(r.polygon([1.1,1.1], [1.9,1.1], [1.9,1.9], [1.1,1.9])).includes(r.polygon([1.1,1.1], [2,1.1], [2,2], [1.1,2])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([2,1.1], [3,1.1], [3,1.9], [2,1.9])) + ot: false + - cd: r.polygon([1,1], [2,1], [2,2], [1,2]).includes(r.polygon([2,2], [3,2], [3,3], [2,3])) + ot: false + # includes on an array/stream + - cd: r.expr([r.polygon([0,0], [1,1], [1,0]), r.polygon([0,1], [1,2], [1,1])]).includes(r.point(0,0)).count() + ot: 1 + # Wrong geometry type arguments (the first one must be a polygon) + - cd: r.point(0,0).includes(r.point(0,0)) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Polygon` but found `Point`.') + - cd: r.line([0,0], [0,1]).includes(r.point(0,0)) + ot: err('ReqlQueryLogicError', 'Expected geometry of type `Polygon` but found `LineString`.') diff --git a/ext/librethinkdbxx/test/upstream/geo/operations.yaml b/ext/librethinkdbxx/test/upstream/geo/operations.yaml new file mode 100644 index 00000000..85e07e93 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/operations.yaml @@ -0,0 +1,97 @@ +desc: Test basic geometry operators +tests: + # Distance + # coerce_to('STRING') because the test utility has some issues with rounding and I'm too lazy to investigate that now. + - cd: r.distance(r.point(-122, 37), r.point(-123, 37)).coerce_to('STRING') + ot: ("89011.26253835332") + - cd: r.distance(r.point(-122, 37), r.point(-122, 36)).coerce_to('STRING') + ot: ("110968.30443995494") + - cd: r.distance(r.point(-122, 37), r.point(-122, 36)).eq(r.distance(r.point(-122, 36), r.point(-122, 37))) + ot: true + - cd: r.point(-122, 37).distance(r.point(-123, 37)).coerce_to('STRING') + ot: ("89011.26253835332") + - def: someDist = r.distance(r.point(-122, 37), r.point(-123, 37)) + js: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'m'})) + py: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='m')) + rb: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'m')) + ot: true + - js: someDist.mul(1.0/1000.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'km'})) + py: someDist.mul(1.0/1000.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='km')) + rb: someDist.mul(1.0/1000.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'km')) + ot: true + - js: someDist.mul(1.0/1609.344).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'mi'})) + py: someDist.mul(1.0/1609.344).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='mi')) + rb: someDist.mul(1.0/1609.344).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'mi')) + ot: true + - js: someDist.mul(1.0/0.3048).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'ft'})) + py: someDist.mul(1.0/0.3048).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='ft')) + rb: someDist.mul(1.0/0.3048).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'ft')) + ot: true + - js: someDist.mul(1.0/1852.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {unit:'nm'})) + py: someDist.mul(1.0/1852.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), unit='nm')) + rb: someDist.mul(1.0/1852.0).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :unit=>'nm')) + ot: true + - js: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), {'geo_system':'WGS84'})) + py: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), geo_system='WGS84')) + rb: someDist.eq(r.distance(r.point(-122, 37), r.point(-123, 37), :geo_system=>'WGS84')) + ot: true + # Mearth is a small planet, just 1/10th of earth's size. + - js: someDist.div(10).eq(r.distance(r.point(-122, 37), r.point(-123, 37), {'geo_system':{'a':637813.7, 'f':(1.0/298.257223563)}})) + py: someDist.div(10).eq(r.distance(r.point(-122, 37), r.point(-123, 37), geo_system={'a':637813.7, 'f':(1.0/298.257223563)})) + rb: someDist.div(10).eq(r.distance(r.point(-122, 37), r.point(-123, 37), :geo_system=>{'a':637813.7, 'f':(1.0/298.257223563)})) + ot: true + - py: r.distance(r.point(-122, 37), r.point(-123, 37), geo_system='unit_sphere').coerce_to('STRING') + rb: r.distance(r.point(-122, 37), r.point(-123, 37), :geo_system=>'unit_sphere').coerce_to('STRING') + js: r.distance(r.point(-122, 37), r.point(-123, 37), {'geo_system':'unit_sphere'}).coerce_to('STRING') + ot: ("0.01393875509649327") + - cd: r.distance(r.point(0, 0), r.point(0, 0)).coerce_to('STRING') + ot: ("0") + # These two give the earth's circumference through the poles + - cd: r.distance(r.point(0, 0), r.point(180, 0)).mul(2).coerce_to('STRING') + ot: ("40007862.917250897") + - cd: r.distance(r.point(0, -90), r.point(0, 90)).mul(2).coerce_to('STRING') + ot: ("40007862.917250897") + - cd: r.distance(r.point(0, 0), r.line([0,0], [0,1])).coerce_to('STRING') + ot: ("0") + - cd: r.distance(r.line([0,0], [0,1]), r.point(0, 0)).coerce_to('STRING') + ot: ("0") + - cd: r.distance(r.point(0, 0), r.line([0.1,0], [1,0])).eq(r.distance(r.point(0, 0), r.point(0.1, 0))) + ot: true + - cd: r.distance(r.point(0, 0), r.line([5,-1], [4,2])).coerce_to('STRING') + ot: ("492471.4990055255") + - cd: r.distance(r.point(0, 0), r.polygon([5,-1], [4,2], [10,10])).coerce_to('STRING') + ot: ("492471.4990055255") + - cd: r.distance(r.point(0, 0), r.polygon([0,-1], [0,1], [10,10])).coerce_to('STRING') + ot: ("0") + - cd: r.distance(r.point(0.5, 0.5), r.polygon([0,-1], [0,1], [10,10])).coerce_to('STRING') + ot: ("0") + + # Fill + - js: r.circle([0,0], 1, {fill:false}).eq(r.circle([0,0], 1, {fill:true})) + py: r.circle([0,0], 1, fill=false).eq(r.circle([0,0], 1, fill=true)) + rb: r.circle([0,0], 1, :fill=>false).eq(r.circle([0,0], 1, :fill=>true)) + ot: false + - js: r.circle([0,0], 1, {fill:false}).fill().eq(r.circle([0,0], 1, {fill:true})) + py: r.circle([0,0], 1, fill=false).fill().eq(r.circle([0,0], 1, fill=true)) + rb: r.circle([0,0], 1, :fill=>false).fill().eq(r.circle([0,0], 1, :fill=>true)) + ot: true + + # Subtraction + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0.1,0.1], [0.9,0.1], [0.9,0.9], [0.1,0.9])) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0],[1,0],[1,1],[0,1],[0,0]],[[0.1,0.1],[0.9,0.1],[0.9,0.9],[0.1,0.9],[0.1,0.1]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0.1,0.9], [0.9,0.0], [0.9,0.9], [0.1,0.9])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,0], [2,0], [2,2], [0,2])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,-2], [1,-2], [-1,1], [0,-1])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,-1], [1,-1], [1,0], [0,0])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0.1,-1], [0.9,-1], [0.9,0.5], [0.1,0.5])) + ot: err('ReqlQueryLogicError', 'The second argument to `polygon_sub` is not contained in the first one.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,0],[0.1,0.9],[0.9,0.9],[0.9,0.1])) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0,0],[1,0],[1,1],[0,1],[0,0]],[[0,0],[0.1,0.9],[0.9,0.9],[0.9,0.1],[0,0]]], 'type':'Polygon'}) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.polygon([0,0],[0.1,0.9],[0.9,0.9],[0.9,0.1]).polygon_sub(r.polygon([0.2,0.2],[0.5,0.8],[0.8,0.2]))) + ot: err('ReqlQueryLogicError', 'Expected a Polygon with only an outer shell. This one has holes.', [0]) + - cd: r.polygon([0,0], [1,0], [1,1], [0,1]).polygon_sub(r.line([0,0],[0.9,0.1],[0.9,0.9],[0.1,0.9])) + ot: err('ReqlQueryLogicError', 'Expected a Polygon but found a LineString.', []) diff --git a/ext/librethinkdbxx/test/upstream/geo/primitives.yaml b/ext/librethinkdbxx/test/upstream/geo/primitives.yaml new file mode 100644 index 00000000..1fd2c0e0 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/geo/primitives.yaml @@ -0,0 +1,50 @@ +desc: Test geometric primitive constructors +tests: + # Circle + - js: r.circle([0,0], 1, {num_vertices:3}) + py: r.circle([0,0], 1, num_vertices=3) + rb: r.circle([0,0], 1, :num_vertices=>3) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle(r.point(0,0), 1, {num_vertices:3}) + py: r.circle(r.point(0,0), 1, num_vertices=3) + rb: r.circle(r.point(0,0), 1, :num_vertices=>3) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle([0,0], 1, {num_vertices:3, fill:false}) + py: r.circle([0,0], 1, num_vertices=3, fill=false) + rb: r.circle([0,0], 1, :num_vertices=>3, :fill=>false) + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]], 'type':'LineString'}) + + - js: r.circle([0,0], 14000000, {num_vertices:3}) + py: r.circle([0,0], 14000000, num_vertices=3) + rb: r.circle([0,0], 14000000, :num_vertices=>3) + ot: err('ReqlQueryLogicError', 'Radius must be smaller than a quarter of the circumference along the minor axis of the reference ellipsoid. Got 14000000m, but must be smaller than 9985163.1855612862855m.', [0]) + + - js: r.circle([0,0], 1, {num_vertices:3, geo_system:'WGS84'}) + py: r.circle([0,0], 1, num_vertices=3, geo_system='WGS84') + rb: r.circle([0,0], 1, :num_vertices=>3, :geo_system=>'WGS84') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle([0,0], 2, {num_vertices:3, geo_system:'unit_'+'sphere'}) + py: r.circle([0,0], 2, num_vertices=3, geo_system='unit_sphere') + rb: r.circle([0,0], 2, :num_vertices=>3, :geo_system=>'unit_sphere') + ot: err('ReqlQueryLogicError', 'Radius must be smaller than a quarter of the circumference along the minor axis of the reference ellipsoid. Got 2m, but must be smaller than 1.570796326794896558m.', [0]) + + - js: r.circle([0,0], 0.1, {num_vertices:3, geo_system:'unit_'+'sphere'}) + py: r.circle([0,0], 0.1, num_vertices=3, geo_system='unit_sphere') + rb: r.circle([0,0], 0.1, :num_vertices=>3, :geo_system=>'unit_sphere') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -5.729577951308232], [-4.966092947444857, 2.861205754495701], [4.966092947444857, 2.861205754495701], [0, -5.729577951308232]]], 'type':'Polygon'}) + testopts: + precision: 0.0000000000001 + + - js: r.circle([0,0], 1.0/1000.0, {num_vertices:3, unit:'km'}) + py: r.circle([0,0], 1.0/1000.0, num_vertices=3, unit='km') + rb: r.circle([0,0], 1.0/1000.0, :num_vertices=>3, :unit=>'km') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + + - js: r.circle([0,0], 1.0/1609.344, {num_vertices:3, unit:'mi'}) + py: r.circle([0,0], 1.0/1609.344, num_vertices=3, unit='mi') + rb: r.circle([0,0], 1.0/1609.344, :num_vertices=>3, :unit=>'mi') + ot: ({'$reql_type$':'GEOMETRY', 'coordinates':[[[0, -9.04369477050382e-06], [-7.779638566553426e-06, 4.5218473852518965e-06], [7.779638566553426e-06, 4.5218473852518965e-06], [0, -9.04369477050382e-06]]], 'type':'Polygon'}) + diff --git a/ext/librethinkdbxx/test/upstream/joins.yaml b/ext/librethinkdbxx/test/upstream/joins.yaml new file mode 100644 index 00000000..80ffed50 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/joins.yaml @@ -0,0 +1,133 @@ +desc: Tests that manipulation data in tables +table_variable_name: tbl, tbl2, senders, receivers, messages, otbl, otbl2 +tests: + + # Setup some more tables + + - py: r.db('test').table_create('test3', primary_key='foo') + rb: r.db('test').table_create('test3', {:primary_key=>'foo'}) + js: r.db('test').tableCreate('test3', {'primaryKey':'foo'}) + ot: partial({'tables_created':1}) + - def: tbl3 = r.db('test').table('test3') + + - py: tbl.insert(r.range(0, 100).map({'id':r.row, 'a':r.row % 4})) + rb: tbl.insert(r.range(0, 100).map{|row| {'id':row, a:row % 4}}) + js: tbl.insert(r.range(0, 100).map(function (row) { return {'id':row, 'a':row.mod(4)}; })) + ot: partial({'errors':0, 'inserted':100}) + + - py: tbl2.insert(r.range(0, 100).map({'id':r.row, 'b':r.row % 4})) + rb: tbl2.insert(r.range(0, 100).map{|row| {'id':row, b:row % 4}}) + js: tbl2.insert(r.range(0, 100).map(function (row) { return {'id':row, 'b':row.mod(4)}; })) + ot: partial({'errors':0, 'inserted':100}) + + - py: tbl3.insert(r.range(0, 100).map({'foo':r.row, 'b':r.row % 4})) + rb: tbl3.insert(r.range(0, 100).map{|row| {'foo':row, b:row % 4}}) + js: tbl3.insert(r.range(0, 100).map(function (row) { return {'foo':row, 'b':row.mod(4)}; })) + ot: partial({'errors':0, 'inserted':100}) + + - py: otbl.insert(r.range(1,100).map({'id': r.row, 'a': r.row})) + - py: otbl2.insert(r.range(1,100).map({'id': r.row, 'b': 2 * r.row})) + + # Inner-Join + + - def: + py: ij = tbl.inner_join(tbl2, lambda x,y:x['a'] == y['b']).zip() + js: ij = tbl.innerJoin(tbl2, function(x, y) { return x('a').eq(y('b')); }).zip() + rb: ij = tbl.inner_join(tbl2){ |x, y| x[:a].eq y[:b] }.zip + - cd: ij.count() + ot: 2500 + - py: ij.filter(lambda row:row['a'] != row['b']).count() + js: ij.filter(function(row) { return row('a').ne(row('b')); }).count() + rb: ij.filter{ |row| row[:a].ne row[:b] }.count + ot: 0 + + # Outer-Join + - def: + py: oj = tbl.outer_join(tbl2, lambda x,y:x['a'] == y['b']).zip() + js: oj = tbl.outerJoin(tbl2, function(x, y) { return x('a').eq(y('b')); }).zip() + rb: oj = tbl.outer_join(tbl2){ |x, y| x[:a].eq y[:b] }.zip + - cd: oj.count() + ot: 2500 + - py: oj.filter(lambda row:row['a'] != row['b']).count() + js: oj.filter(function(row) { return row('a').ne(row('b')); }).count() + rb: oj.filter{ |row| row[:a].ne row[:b] }.count + ot: 0 + + # Ordered eq_join + - py: blah = otbl.order_by("id").eq_join(r.row['id'], otbl2, ordered=True).zip() + ot: [{'id': i, 'a': i, 'b': i * 2} for i in range(1, 100)] + - py: blah = otbl.order_by(r.desc("id")).eq_join(r.row['id'], otbl2, ordered=True).zip() + ot: [{'id': i, 'a': i, 'b': i * 2} for i in range(99, 0, -1)] + - py: blah = otbl.order_by("id").eq_join(r.row['a'], otbl2, ordered=True).zip() + ot: [{'id': i, 'a': i, 'b': i * 2} for i in range(1, 100)] + + # Eq-Join + - cd: tbl.eq_join('a', tbl2).zip().count() + ot: 100 + + - cd: tbl.eq_join('fake', tbl2).zip().count() + ot: 0 + + - py: tbl.eq_join(lambda x:x['a'], tbl2).zip().count() + rb: tbl.eq_join(lambda{|x| x['a']}, tbl2).zip().count() + js: tbl.eq_join(function(x) { return x('a'); }, tbl2).zip().count() + ot: 100 + + - py: tbl.eq_join(lambda x:x['fake'], tbl2).zip().count() + rb: tbl.eq_join(lambda{|x| x['fake']}, tbl2).zip().count() + js: tbl.eq_join(function(x) { return x('fake'); }, tbl2).zip().count() + ot: 0 + + - py: tbl.eq_join(lambda x:null, tbl2).zip().count() + rb: tbl.eq_join(lambda{|x| null}, tbl2).zip().count() + js: tbl.eq_join(function(x) { return null; }, tbl2).zip().count() + ot: 0 + + - py: tbl.eq_join(lambda x:x['a'], tbl2).count() + rb: tbl.eq_join(lambda {|x| x[:a]}, tbl2).count() + js: tbl.eq_join(function(x) { return x('a'); }, tbl2).count() + ot: 100 + + # eqjoin where id isn't a primary key + - cd: tbl.eq_join('a', tbl3).zip().count() + ot: 100 + + - py: tbl.eq_join(lambda x:x['a'], tbl3).count() + rb: tbl.eq_join(lambda {|x| x[:a]}, tbl3).count() + js: tbl.eq_join(function(x) { return x('a'); }, tbl3).count() + ot: 100 + + # eq_join with r.row + - py: tbl.eq_join(r.row['a'], tbl2).count() + js: tbl.eq_join(r.row('a'), tbl2).count() + ot: 100 + + # test an inner-join condition where inner-join differs from outer-join + - def: left = r.expr([{'a':1},{'a':2},{'a':3}]) + - def: right = r.expr([{'b':2},{'b':3}]) + + - py: left.inner_join(right, lambda l, r:l['a'] == r['b']).zip() + js: left.innerJoin(right, function(l, r) { return l('a').eq(r('b')); }).zip() + rb: left.inner_join(right){ |lt, rt| lt[:a].eq(rt[:b]) }.zip + ot: [{'a':2,'b':2},{'a':3,'b':3}] + + # test an outer-join condition where outer-join differs from inner-join + - py: left.outer_join(right, lambda l, r:l['a'] == r['b']).zip() + js: left.outerJoin(right, function(l, r) { return l('a').eq(r('b')); }).zip() + rb: left.outer_join(right){ |lt, rt| lt[:a].eq(rt[:b]) }.zip + ot: [{'a':1},{'a':2,'b':2},{'a':3,'b':3}] + + - rb: senders.insert({id:1, sender:'Sender One'})['inserted'] + ot: 1 + - rb: receivers.insert({id:1, receiver:'Receiver One'})['inserted'] + ot: 1 + - rb: messages.insert([{id:10, sender_id:1, receiver_id:1, msg:'Message One'}, {id:20, sender_id:1, receiver_id:1, msg:'Message Two'}, {id:30, sender_id:1, receiver_id:1, msg:'Message Three'}])['inserted'] + ot: 3 + + - rb: messages.orderby(index:'id').eq_join('sender_id', senders).without({right:{id:true}}).zip.eq_join('receiver_id', receivers).without({right:{id:true}}).zip + ot: [{'id':10,'msg':'Message One','receiver':'Receiver One','receiver_id':1,'sender':'Sender One','sender_id':1},{'id':20,'msg':'Message Two','receiver':'Receiver One','receiver_id':1,'sender':'Sender One','sender_id':1},{'id':30,'msg':'Message Three','receiver':'Receiver One','receiver_id':1,'sender':'Sender One','sender_id':1}] + + # Clean up + + - cd: r.db('test').table_drop('test3') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/json.yaml b/ext/librethinkdbxx/test/upstream/json.yaml new file mode 100644 index 00000000..2a896cd4 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/json.yaml @@ -0,0 +1,74 @@ +desc: Tests RQL json parsing +tests: + + - cd: r.json("[1,2,3]") + ot: [1,2,3] + + - cd: r.json("1") + ot: 1 + + - cd: r.json("{}") + ot: {} + + - cd: r.json('"foo"') + ot: "foo" + + - cd: r.json("[1,2") + ot: err("ReqlQueryLogicError", 'Failed to parse "[1,2" as JSON:' + ' Missing a comma or \']\' after an array element.', [0]) + + - cd: r.json("[1,2,3]").to_json_string() + ot: '[1,2,3]' + + - js: r.json("[1,2,3]").toJSON() + py: r.json("[1,2,3]").to_json() + ot: '[1,2,3]' + + - cd: r.json("{\"foo\":4}").to_json_string() + ot: '{"foo":4}' + + - js: r.json("{\"foo\":4}").toJSON() + py: r.json("{\"foo\":4}").to_json() + ot: '{"foo":4}' + + # stress test: data is from http://www.mockaroo.com/ + - def: text = '[{"id":1,"first_name":"Harry","last_name":"Riley","email":"hriley0@usgs.gov","country":"Andorra","ip_address":"221.25.65.136"},{"id":2,"first_name":"Bonnie","last_name":"Anderson","email":"banderson1@list-manage.com","country":"Tuvalu","ip_address":"116.162.43.150"},{"id":3,"first_name":"Marie","last_name":"Schmidt","email":"mschmidt2@diigo.com","country":"Iraq","ip_address":"181.105.59.57"},{"id":4,"first_name":"Phillip","last_name":"Willis","email":"pwillis3@com.com","country":"Montenegro","ip_address":"24.223.139.156"}]' + - def: sorted = '[{"country":"Andorra","email":"hriley0@usgs.gov","first_name":"Harry","id":1,"ip_address":"221.25.65.136","last_name":"Riley"},{"country":"Tuvalu","email":"banderson1@list-manage.com","first_name":"Bonnie","id":2,"ip_address":"116.162.43.150","last_name":"Anderson"},{"country":"Iraq","email":"mschmidt2@diigo.com","first_name":"Marie","id":3,"ip_address":"181.105.59.57","last_name":"Schmidt"},{"country":"Montenegro","email":"pwillis3@com.com","first_name":"Phillip","id":4,"ip_address":"24.223.139.156","last_name":"Willis"}]' + + - cd: r.json(text).to_json_string() + ot: sorted + + - cd: r.expr(r.minval).to_json_string() + ot: err('ReqlQueryLogicError', 'Cannot convert `r.minval` to JSON.') + + - cd: r.expr(r.maxval).to_json_string() + ot: err('ReqlQueryLogicError', 'Cannot convert `r.maxval` to JSON.') + + - cd: r.expr(r.minval).coerce_to('string') + ot: err('ReqlQueryLogicError', 'Cannot convert `r.minval` to JSON.') + + - cd: r.expr(r.maxval).coerce_to('string') + ot: err('ReqlQueryLogicError', 'Cannot convert `r.maxval` to JSON.') + + - cd: r.time(2014,9,11, 'Z') + runopts: + time_format: 'raw' + ot: {'timezone':'+00:00','$reql_type$':'TIME','epoch_time':1410393600} + + - cd: r.time(2014,9,11, 'Z').to_json_string() + ot: '{"$reql_type$":"TIME","epoch_time":1410393600,"timezone":"+00:00"}' + + - cd: r.point(0,0) + ot: {'$reql_type$':'GEOMETRY','coordinates':[0,0],'type':'Point'} + + - cd: r.point(0,0).to_json_string() + ot: '{"$reql_type$":"GEOMETRY","coordinates":[0,0],"type":"Point"}' + + - def: + rb: s = "\x66\x6f\x6f".force_encoding('BINARY') + py: s = b'\x66\x6f\x6f' + js: s = Buffer("\x66\x6f\x6f", 'binary') + - cd: r.binary(s) + ot: s + + - cd: r.expr("foo").coerce_to("binary").to_json_string() + ot: '{"$reql_type$":"BINARY","data":"Zm9v"}' diff --git a/ext/librethinkdbxx/test/upstream/limits.yaml b/ext/librethinkdbxx/test/upstream/limits.yaml new file mode 100644 index 00000000..41316df2 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/limits.yaml @@ -0,0 +1,128 @@ +desc: Tests array limit variations +table_variable_name: tbl +tests: + + # test simplistic array limits + - cd: r.expr([1,1,1,1]).union([1, 1, 1, 1]) + runopts: + array_limit: 8 + ot: [1,1,1,1,1,1,1,1] + - cd: r.expr([1,2,3,4]).union([5, 6, 7, 8]) + runopts: + array_limit: 4 + ot: err("ReqlResourceLimitError", "Array over size limit `4`.", [0]) + + # test array limits on query creation + - cd: r.expr([1,2,3,4,5,6,7,8]) + runopts: + array_limit: 4 + ot: err("ReqlResourceLimitError", "Array over size limit `4`.", [0]) + + # test bizarre array limits + - cd: r.expr([1,2,3,4,5,6,7,8]) + runopts: + array_limit: -1 + ot: err("ReqlQueryLogicError", "Illegal array size limit `-1`. (Must be >= 1.)", []) + + - cd: r.expr([1,2,3,4,5,6,7,8]) + runopts: + array_limit: 0 + ot: err("ReqlQueryLogicError", "Illegal array size limit `0`. (Must be >= 1.)", []) + + # make enormous > 100,000 element array + - def: ten_l = r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - def: + js: ten_f = function(l) { return ten_l } + py: ten_f = lambda l:list(range(1,11)) + - def: + js: huge_l = r.expr(ten_l).concatMap(ten_f).concatMap(ten_f).concatMap(ten_f).concatMap(ten_f) + py: huge_l = r.expr(ten_l).concat_map(ten_f).concat_map(ten_f).concat_map(ten_f).concat_map(ten_f) + rb: huge_l = r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l} + - cd: huge_l.append(1).count() + runopts: + array_limit: 100001 + ot: 100001 + + # attempt to insert enormous array + - cd: tbl.insert({'id':0, 'array':huge_l.append(1)}) + runopts: + array_limit: 100001 + ot: partial({'errors':1, 'first_error':"Array too large for disk writes (limit 100,000 elements)."}) + + - cd: tbl.get(0) + runopts: + array_limit: 100001 + ot: (null) + + # attempt to read array that violates limit from disk + - cd: tbl.insert({'id':1, 'array':ten_l}) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1}) + - cd: tbl.get(1) + runopts: + array_limit: 4 + ot: ({'array':[1,2,3,4,5,6,7,8,9,10],'id':1}) + + + # Test that the changefeed queue size actually causes changes to be sent early. + - cd: tbl.delete().get_field('deleted') + ot: 1 + + - cd: c = tbl.changes({squash:1000000, changefeed_queue_size:10}) + py: c = tbl.changes(squash=1000000, changefeed_queue_size=10) + + - cd: tbl.insert([{'id':0}, {'id':1}, {'id':2}, {'id':3}, {'id':4}, {'id':5}, {'id':6}]).get_field('inserted') + ot: 7 + - py: fetch(c, 7) + rb: fetch(c, 7) + ot: bag([{'old_val':null, 'new_val':{'id':0}}, + {'old_val':null, 'new_val':{'id':1}}, + {'old_val':null, 'new_val':{'id':2}}, + {'old_val':null, 'new_val':{'id':3}}, + {'old_val':null, 'new_val':{'id':4}}, + {'old_val':null, 'new_val':{'id':5}}, + {'old_val':null, 'new_val':{'id':6}}]) + + - cd: tbl.insert([{'id':7}, {'id':8}, {'id':9}, {'id':10}, {'id':11}, {'id':12}, {'id':13}]).get_field('inserted') + ot: 7 + - py: fetch(c, 7) + rb: fetch(c, 7) + ot: bag([{'old_val':null, 'new_val':{'id':7}}, + {'old_val':null, 'new_val':{'id':8}}, + {'old_val':null, 'new_val':{'id':9}}, + {'old_val':null, 'new_val':{'id':10}}, + {'old_val':null, 'new_val':{'id':11}}, + {'old_val':null, 'new_val':{'id':12}}, + {'old_val':null, 'new_val':{'id':13}}]) + + - cd: tbl.delete().get_field('deleted') + ot: 14 + + - cd: c2 = tbl.changes({squash:1000000}) + py: c2 = tbl.changes(squash=1000000) + runopts: + changefeed_queue_size: 10 + + + - cd: tbl.insert([{'id':0}, {'id':1}, {'id':2}, {'id':3}, {'id':4}, {'id':5}, {'id':6}]).get_field('inserted') + ot: 7 + - py: fetch(c2, 7) + rb: fetch(c2, 7) + ot: bag([{'old_val':null, 'new_val':{'id':0}}, + {'old_val':null, 'new_val':{'id':1}}, + {'old_val':null, 'new_val':{'id':2}}, + {'old_val':null, 'new_val':{'id':3}}, + {'old_val':null, 'new_val':{'id':4}}, + {'old_val':null, 'new_val':{'id':5}}, + {'old_val':null, 'new_val':{'id':6}}]) + + - cd: tbl.insert([{'id':7}, {'id':8}, {'id':9}, {'id':10}, {'id':11}, {'id':12}, {'id':13}]).get_field('inserted') + ot: 7 + - py: fetch(c2, 7) + rb: fetch(c2, 7) + ot: bag([{'old_val':null, 'new_val':{'id':7}}, + {'old_val':null, 'new_val':{'id':8}}, + {'old_val':null, 'new_val':{'id':9}}, + {'old_val':null, 'new_val':{'id':10}}, + {'old_val':null, 'new_val':{'id':11}}, + {'old_val':null, 'new_val':{'id':12}}, + {'old_val':null, 'new_val':{'id':13}}]) diff --git a/ext/librethinkdbxx/test/upstream/match.yaml b/ext/librethinkdbxx/test/upstream/match.yaml new file mode 100644 index 00000000..55e33801 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/match.yaml @@ -0,0 +1,38 @@ +desc: Tests for match +table_variable_name: tbl +tests: + - cd: r.expr("abcdefg").match("a(b.e)|b(c.e)") + ot: ({'str':'bcde','groups':[null,{'start':2,'str':'cde','end':5}],'start':1,'end':5}) + - cd: r.expr("abcdefg").match("a(b.e)|B(c.e)") + ot: (null) + - cd: r.expr("abcdefg").match("(?i)a(b.e)|B(c.e)") + ot: ({'str':'bcde','groups':[null,{'start':2,'str':'cde','end':5}],'start':1,'end':5}) + + - cd: r.expr(["aba", "aca", "ada", "aea"]).filter{|row| row.match("a(.)a")[:groups][0][:str].match("[cd]")} + py: r.expr(["aba", "aca", "ada", "aea"]).filter(lambda row:row.match("a(.)a")['groups'][0]['str'].match("[cd]")) + js: r.expr(["aba", "aca", "ada", "aea"]).filter(function(row){return row.match("a(.)a")('groups').nth(0)('str').match("[cd]")}) + ot: (["aca", "ada"]) + + - cd: tbl.insert([{'id':0,'a':'abc'},{'id':1,'a':'ab'},{'id':2,'a':'bc'}]) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':3}) + + - cd: tbl.filter{|row| row['a'].match('b')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('b')).order_by('id') + js: tbl.filter(function(row){return row('a').match('b')}).order_by('id') + ot: ([{'id':0,'a':'abc'},{'id':1,'a':'ab'},{'id':2,'a':'bc'}]) + - cd: tbl.filter{|row| row['a'].match('ab')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('ab')).order_by('id') + js: tbl.filter(function(row){return row('a').match('ab')}).order_by('id') + ot: ([{'id':0,'a':'abc'},{'id':1,'a':'ab'}]) + - cd: tbl.filter{|row| row['a'].match('ab$')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('ab$')).order_by('id') + js: tbl.filter(function(row){return row('a').match('ab$')}).order_by('id') + ot: ([{'id':1,'a':'ab'}]) + - cd: tbl.filter{|row| row['a'].match('^b$')}.orderby('id') + py: tbl.filter(lambda row:row['a'].match('^b$')).order_by('id') + js: tbl.filter(function(row){return row('a').match('^b$')}).order_by('id') + ot: ([]) + + - cd: r.expr("").match("ab\\9") + ot: | + err("ReqlQueryLogicError", "Error in regexp `ab\\9` (portion `\\9`): invalid escape sequence: \\9", []) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/add.yaml b/ext/librethinkdbxx/test/upstream/math_logic/add.yaml new file mode 100644 index 00000000..e956aaa7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/add.yaml @@ -0,0 +1,65 @@ +desc: Tests for basic usage of the add operation +tests: + - cd: r.add(1, 1) + ot: 2 + + - js: r(1).add(1) + py: + - r.expr(1) + 1 + - 1 + r.expr(1) + - r.expr(1).add(1) + rb: + - r(1) + 1 + - r(1).add(1) + ot: 2 + + - py: r.expr(-1) + 1 + js: r(-1).add(1) + rb: (r -1) + 1 + ot: 0 + + - py: r.expr(1.75) + 8.5 + js: r(1.75).add(8.5) + rb: (r 1.75) + 8.5 + ot: 10.25 + + # Add is polymorphic on strings + - py: r.expr('') + '' + js: r('').add('') + rb: (r '') + '' + ot: '' + + - py: r.expr('abc') + 'def' + js: r('abc').add('def') + rb: (r 'abc') + 'def' + ot: 'abcdef' + + # Add is polymorphic on arrays + - cd: r.expr([1,2]) + [3] + [4,5] + [6,7,8] + js: r([1,2]).add([3]).add([4,5]).add([6,7,8]) + ot: [1,2,3,4,5,6,7,8] + + # All arithmetic operations (except mod) actually support arbitrary arguments + # but this feature can't be accessed in Python because it's operators are binary + - js: r(1).add(2,3,4,5) + ot: 15 + + - js: r('a').add('b', 'c', 'd') + ot: 'abcd' + + # Type errors + - cd: r(1).add('a') + py: r.expr(1) + 'a' + rb: r(1) + 'a' + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", [1]) + + - cd: r('a').add(1) + py: r.expr('a') + 1 + rb: r('a') + 1 + ot: err("ReqlQueryLogicError", "Expected type STRING but found NUMBER.", [1]) + + - cd: r([]).add(1) + py: r.expr([]) + 1 + rb: r([]) + 1 + ot: err("ReqlQueryLogicError", "Expected type ARRAY but found NUMBER.", [1]) + diff --git a/ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml b/ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml new file mode 100644 index 00000000..f1181f87 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/aliases.yaml @@ -0,0 +1,46 @@ +desc: Test named aliases for math and logic operators +tests: + + - cd: + - r.expr(0).add(1) + - r.add(0, 1) + - r.expr(2).sub(1) + - r.sub(2, 1) + - r.expr(2).div(2) + - r.div(2, 2) + - r.expr(1).mul(1) + - r.mul(1, 1) + - r.expr(1).mod(2) + - r.mod(1, 2) + ot: 1 + + - cd: + - r.expr(True).and(True) + - r.expr(True).or(True) + - r.and(True, True) + - r.or(True, True) + - r.expr(False).not() + - r.not(False) + py: + - r.expr(True).and_(True) + - r.expr(True).or_(True) + - r.and_(True, True) + - r.or_(True, True) + - r.expr(False).not_() + - r.not_(False) + ot: True + + - cd: + - r.expr(1).eq(1) + - r.expr(1).ne(2) + - r.expr(1).lt(2) + - r.expr(1).gt(0) + - r.expr(1).le(1) + - r.expr(1).ge(1) + - r.eq(1, 1) + - r.ne(1, 2) + - r.lt(1, 2) + - r.gt(1, 0) + - r.le(1, 1) + - r.ge(1, 1) + ot: True diff --git a/ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml b/ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml new file mode 100644 index 00000000..3fed54a9 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/comparison.yaml @@ -0,0 +1,477 @@ +desc: Tests of comparison operators +tests: + + ### Numeric comparisons + + ## basic < + + - cd: r(1).lt(2) + py: + - r.expr(1) < 2 + - 1 < r.expr(2) + - r.expr(1).lt(2) + rb: + - r(1) < 2 + - r(1).lt(2) + - 1 < r(2) + ot: true + - cd: r(3).lt(2) + py: r.expr(3) < 2 + rb: r(3) < 2 + ot: false + - py: r.expr(2) < 2 + js: r(2).lt(2) + rb: r(2) < 2 + ot: false + + # All Comparisons can take an arbitrary number of arguments though + # the functionality is only available in JS at the moment + - js: r(1).lt(2, 3, 4) + ot: true + - js: r(1).lt(2, 3, 2) + ot: false + + ## basic > + + - cd: r(1).gt(2) + py: + - r.expr(1) > 2 + - 1 > r.expr(2) + - r.expr(1).gt(2) + rb: + - r(1) > 2 + - r(1).gt(2) + ot: false + - py: r.expr(3) > 2 + js: r(3).gt(2) + rb: r(3) > 2 + ot: true + - py: r.expr(2) > 2 + js: r(2).gt(2) + rb: r(2) > 2 + ot: false + + - js: r(4).gt(3, 2, 1) + ot: true + - js: r(4).gt(3, 2, 3) + ot: false + + ## basic == + + - cd: r(1).eq(2) + py: + - r.expr(1) == 2 + - 1 == r.expr(2) + - r.expr(1).eq(2) + rb: r(1).eq 2 + ot: false + - py: r.expr(3) == 2 + js: r(3).eq(2) + rb: r(3).eq 2 + ot: false + - py: r.expr(2) == 2 + js: r(2).eq(2) + rb: r(2).eq 2 + ot: true + + - js: r(1).eq(1, 1, 1) + ot: true + - js: r(1).eq(1, 2, 1) + ot: false + + ## basic != + + - cd: r(1).ne(2) + py: + - r.expr(1) != 2 + - 1 != r.expr(2) + - r.expr(1).ne(2) + rb: r(1).ne 2 + ot: true + - py: r.expr(3) != 2 + js: r(3).ne(2) + rb: r(3).ne 2 + ot: true + - py: r.expr(2) != 2 + js: r(2).ne(2) + rb: r(2).ne 2 + ot: false + + - js: r(1).ne(3, 2, 4) + ot: true + - js: r(1).ne(3, 2, 3) + ot: true + + ## basic <= + + - js: r(1).le(2) + py: + - r.expr(1) <= 2 + - 1 <= r.expr(2) + - r.expr(1).le(2) + rb: + - r(1) <= 2 + - r(1).le(2) + ot: true + - py: r.expr(3) <= 2 + js: r(3).le(2) + rb: r(3) <= 2 + ot: false + - py: r.expr(2) <= 2 + js: r(2).le(2) + rb: r(2) <= 2 + ot: true + + - js: r(1).le(1, 2, 2) + ot: true + - js: r(1).le(1, 3, 2) + ot: false + + ## basic >= + + - cd: r(1).ge(2) + py: + - r.expr(1) >= 2 + - 1 >= r.expr(2) + - r.expr(1).ge(2) + rb: + - r(1) >= 2 + - r(1).ge(2) + ot: false + - py: r.expr(3) >= 2 + js: r(3).ge(2) + rb: r(3) >= 2 + ot: true + - py: r.expr(2) >= 2 + js: r(2).ge(2) + rb: r(2) >= 2 + ot: true + + - js: r(4).ge(4, 2, 2) + ot: true + - js: r(4).ge(4, 2, 3) + ot: false + + # Comparisons for NULL + - cd: r(null).eq(null) + py: + - r.expr(null) == null + - null == r.expr(null) + ot: true + + - cd: r(null).lt(null) + py: + - r.expr(null) < null + - null < r.expr(null) + - r.expr(null).lt(null) + rb: r(null) < null + ot: false + + - cd: r(null).gt(null) + py: + - r.expr(null) > null + - null > r.expr(null) + - r.expr(null).gt(null) + rb: r(null) > null + ot: false + + # Comparisons for STRING + # STRING comparison should be lexicagraphical + - py: r.expr('a') == 'a' + cd: r('a').eq('a') + ot: true + + - py: r.expr('a') == 'aa' + cd: r('a').eq('aa') + ot: false + + - py: r.expr('a') < 'aa' + cd: r('a').lt('aa') + ot: true + + - py: r.expr('a') < 'bb' + cd: r('a').lt('bb') + ot: true + + - py: r.expr('bb') > 'a' + cd: r('bb').gt('a') + ot: true + + - py: r.expr('abcdef') < 'abcdeg' + cd: r('abcdef').lt('abcdeg') + ot: true + + - py: r.expr('abcdefg') > 'abcdeg' + cd: r('abcdefg').gt('abcdeg') + ot: false + + - py: r.expr('A quick brown fox') > 'A quick brawn fox' + js: r('A quick brown fox').gt('A quick brawn fox') + rb: r('A quick brown fox') > 'A quick brawn fox' + ot: true + + # Comparisons for ARRAY + # Also lexicographical + + - py: r.expr([1]) < [2] + js: r([1]).lt([2]) + rb: r([1]) < [2] + ot: true + + - py: r.expr([1]) > [2] + js: r([1]).gt([2]) + rb: r([1]) > [2] + ot: false + + - py: r.expr([1, 0]) < [2] + js: r([1, 0]).lt([2]) + rb: r([1, 0]) < [2] + ot: true + + - py: r.expr([1, 0]) < [1] + js: r([1, 0]).lt([1]) + rb: r([1, 0]) < [1] + ot: false + + - py: r.expr([1, 0]) > [0] + js: r([1, 0]).gt([0]) + rb: r([1, 0]) > [0] + ot: true + + - py: r.expr([1, 'a']) < [1, 'b'] + js: r([1, 'a']).lt([1, 'b']) + rb: r([1, 'a']) < [1, 'b'] + ot: true + + - py: r.expr([0, 'z']) < [1, 'b'] + js: r([0, 'z']).lt([1, 'b']) + rb: r([0, 'z']) < [1, 'b'] + ot: true + + - py: r.expr([1, 1, 1]) < [1, 0, 2] + js: r([1, 1, 1]).lt([1, 0, 2]) + rb: r([1, 1, 1]) < [1, 0, 2] + ot: false + + - py: r.expr([1, 0, 2]) < [1, 1, 1] + js: r([1, 0, 2]).lt([1, 1, 1]) + rb: r([1, 0, 2]) < [1, 1, 1] + ot: true + + # Comparisons for OBJECT + + - py: r.expr({'a':0}) == {'a':0} + cd: r({'a':0}).eq({'a':0}) + ot: true + + - py: r.expr({'a':0, 'b':1}) == {'b':1, 'a':0} + cd: r({'a':0, 'b':1}).eq({'b':1, 'a':0}) + ot: true + + - py: r.expr({'a':0, 'b':1, 'c':2}) == {'b':1, 'a':0} + cd: r({'a':0, 'b':1, 'c':2}).eq({'b':1, 'a':0}) + ot: false + + - py: r.expr({'a':0, 'b':1}) == {'b':1, 'a':0, 'c':2} + cd: r({'a':0, 'b':1}).eq({'b':1, 'a':0, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'd':2}) == {'b':1, 'a':0, 'c':2} + cd: r({'a':0, 'b':1, 'd':2}).eq({'b':1, 'a':0, 'c':2}) + ot: false + + - py: r.expr({'a':0}) < {'b':0} + cd: r({'a':0}).lt({'b':0}) + ot: true + + - py: r.expr({'a':1}) < {'b':0} + cd: r({'a':1}).lt({'b':0}) + ot: true + + - py: r.expr({'b':1}) < {'b':0} + cd: r({'b':1}).lt({'b':0}) + ot: false + + - py: r.expr({'b':1}) < {'a':0} + cd: r({'b':1}).lt({'a':0}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'c':2}) < {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'b':1, 'c':2}).lt({'a':0, 'b':1, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'c':2, 'd':3}) < {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'b':1, 'c':2, 'd':3}).lt({'a':0, 'b':1, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'b':1, 'c':2}) < {'a':0, 'b':1, 'c':2, 'd':3} + cd: r({'a':0, 'b':1, 'c':2}).lt({'a':0, 'b':1, 'c':2, 'd':3}) + ot: true + + - py: r.expr({'a':0, 'c':2}) < {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'c':2}).lt({'a':0, 'b':1, 'c':2}) + ot: false + + - py: r.expr({'a':0, 'c':2}) > {'a':0, 'b':1, 'c':2} + cd: r({'a':0, 'c':2}).gt({'a':0, 'b':1, 'c':2}) + ot: true + + # Comparisons across types + # RQL primitive types compare as if mapped to the following numbers + # MINVAL: 0 + # ARRAY: 1 + # BINARY: 2 + # BOOLEAN: 3 + # NULL: 4 + # NUMBER: 5 + # OBJECT: 6 + # STRING: 7 + # MAXVAL: 8 + - def: + py: everything = r.expr([[],r.now(),r.binary(b"\x00"),false,null,-5,{},"a",r.maxval]) + js: everything = r.expr([[],r.now(),r.binary(Buffer("\x00")),false,null,-5,{},"a",r.maxval]) + rb: everything = r.expr([[],r.now(),r.binary("\x00"),false,null,-5,{},"a",r.maxval]) + + - js: r.and(r.args(everything.map(r.lt(r.minval, r.row)))) + py: r.and_(r.args(everything.map(r.lt(r.minval, r.row)))) + rb: r.and(r.args(everything.map{|x| r.lt(r.minval, x)})) + ot: true + + - js: r.or(r.args(everything.map(r.gt(r.minval, r.row)))) + py: r.or_(r.args(everything.map(r.gt(r.minval, r.row)))) + rb: r.or(r.args(everything.map{|x| r.gt(r.minval, x)})) + ot: false + + - cd: r.eq(r.minval, r.minval) + ot: true + + - py: r.expr([]) < True + js: r([]).lt(true) + rb: r([]) < true + ot: true + + - py: r.expr([1,2]) < False + js: r([1,2]).lt(false) + rb: r([1,2]) < false + ot: true + + - py: r.expr(False) < [] + js: r(false).lt([]) + rb: r(false) < [] + ot: false + + - py: r.expr([]) < r.binary(b"\xAE") + js: r([]).lt(r.binary(Buffer("\x00"))) + rb: r([]) < r.binary("") + ot: true + + - py: r.expr([1,2]) < r.binary(b"\xAE") + js: r([1,2]).lt(r.binary(Buffer("\x00"))) + rb: r([1,2]) < r.binary("") + ot: true + + - py: True < r.expr(null) + js: r(true).lt(null) + rb: r(true) < null + ot: true + + - py: r.expr(null) > [] + js: r(null).gt([]) + rb: r(null) > [] + ot: true + + - py: r.expr(null) < 12 + js: r(null).lt(12) + rb: r(null) < 12 + ot: true + + - py: r.expr(null) < -2 + js: r(null).lt(-2) + rb: r(null) < -2 + ot: true + + - py: r.expr(-12) < {} + js: r(-12).lt({}) + rb: r(-12) < {} + ot: true + + - py: r.expr(100) < {'a':-12} + js: r(100).lt({a:-12}) + rb: r(100) < { :a => 12 } + ot: true + + - py: r.expr(r.binary(b"\xAE")) < 12 + js: r(r.binary(Buffer("\x00"))).lt(12) + rb: r(r.binary("")) < 12 + ot: false + + - py: r.binary(b"0xAE") < 'abc' + js: r.binary(Buffer("0x00")).lt('abc') + rb: r.binary("") < 'abc' + ot: true + + - py: r.binary(b"0xAE") > r.now() + js: r.binary(Buffer("0x00")).gt(r.now()) + rb: r.binary("") > r.now() + ot: false + + - cd: r.now() > 12 + js: r.now().gt(12) + ot: true + + - cd: r.now() > 'abc' + js: r.now().gt('abc') + ot: false + + - py: r.expr("abc") > {'a':-12} + js: r('abc').gt({a:-12}) + rb: r('abc') > { :a => 12 } + ot: true + + - py: r.expr("abc") > {'abc':'abc'} + js: r('abc').gt({abc:'abc'}) + rb: r('abc') > { :abc => 'abc' } + ot: true + + - py: r.expr('zzz') > 128 + js: r('zzz').gt(128) + rb: r('zzz') > 128 + ot: true + + - py: r.expr('zzz') > {} + js: r('zzz').gt({}) + rb: r('zzz') > {} + ot: true + + - py: 'zzz' > r.expr(-152) + js: r('zzz').gt(-152) + rb: r('zzz') > -152 + ot: true + + - py: 'zzz' > r.expr(null) + js: r('zzz').gt(null) + rb: r('zzz') > null + ot: true + + - py: 'zzz' > r.expr([]) + js: r('zzz').gt([]) + rb: r('zzz') > [] + ot: true + + - def: + rb: everything2 = r.expr([r.minval,[],r.now(),r.binary("\x00"),false,null,-5,{},"a"]) + py: everything2 = r.expr([r.minval,[],r.now(),r.binary(b"\x00"),false,null,-5,{},"a"]) + js: everything2 = r.expr([r.minval,[],r.now(),r.binary(Buffer("\x00")),false,null,-5,{},"a"]) + + - js: r.and(r.args(everything2.map(r.gt(r.maxval, r.row)))) + py: r.and_(r.args(everything2.map(r.gt(r.maxval, r.row)))) + rb: r.and(r.args(everything2.map{|x| r.gt(r.maxval, x)})) + ot: true + + - js: r.or(r.args(everything2.map(r.lt(r.maxval, r.row)))) + py: r.or_(r.args(everything2.map(r.lt(r.maxval, r.row)))) + rb: r.or(r.args(everything2.map{|x| r.lt(r.maxval, x)})) + ot: false + + - cd: r.eq(r.maxval, r.maxval) + ot: true diff --git a/ext/librethinkdbxx/test/upstream/math_logic/div.yaml b/ext/librethinkdbxx/test/upstream/math_logic/div.yaml new file mode 100644 index 00000000..ffca4156 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/div.yaml @@ -0,0 +1,52 @@ +desc: Tests for the basic usage of the division operation +tests: + + - cd: r(4).div(2) + py: + - r.expr(4) / 2 + - 4 / r.expr(2) + - r.expr(4).div(2) + rb: + - (r 4) / 2 + - r(4).div 2 + - 4 / r(2) + ot: 2 + + - py: r.expr(-1) / -2 + js: r(-1).div(-2) + rb: (r -1) / -2 + ot: 0.5 + + - py: r.expr(4.9) / 0.7 + js: r(4.9).div(0.7) + rb: (r 4.9) / 0.7 + ot: 4.9 / 0.7 + + - cd: r.expr(1).div(2,3,4,5) + ot: 1.0/120 + + # Divide by zero test + - cd: + - r(1).div(0) + - r(2.0).div(0) + - r(3).div(0.0) + - r(4.0).div(0.0) + - r(0).div(0) + - r(0.0).div(0.0) + py: + - r.expr(1) / 0 + - r.expr(2.0) / 0 + - r.expr(3) / 0.0 + - r.expr(4.0) / 0.0 + - r.expr(0) / 0 + - r.expr(0.0) / 0.0 + ot: err('ReqlQueryLogicError', 'Cannot divide by zero.', [1]) + + # Type errors + - py: r.expr('a') / 0.8 + cd: r('a').div(0.8) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - py: r.expr(1) / 'a' + cd: r(1).div('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml b/ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml new file mode 100644 index 00000000..d32526f5 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/floor_ceil_round.yaml @@ -0,0 +1,114 @@ +desc: tests for `floor`, `ceil`, and `round`, tests inspired by the Python test suite +tests: + - cd: r.floor(1.0).type_of() + ot: "NUMBER" + - cd: r.floor(1.0) + ot: 1.0 + - cd: r.expr(1.0).floor() + ot: 1.0 + + - cd: r.floor(0.5) + ot: 0.0 + - cd: r.floor(1.0) + ot: 1.0 + - cd: r.floor(1.5) + ot: 1.0 + - cd: r.floor(-0.5) + ot: -1.0 + - cd: r.floor(-1.0) + ot: -1.0 + - cd: r.floor(-1.5) + ot: -2.0 + + - cd: r.expr('X').floor() + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + + - cd: r.ceil(1.0).type_of() + ot: "NUMBER" + - cd: r.ceil(1.0) + ot: 1.0 + - cd: r.expr(1.0).ceil() + ot: 1.0 + + - cd: r.ceil(0.5) + ot: 1.0 + - cd: r.ceil(1.0) + ot: 1.0 + - cd: r.ceil(1.5) + ot: 2.0 + - cd: r.ceil(-0.5) + ot: 0.0 + - cd: r.ceil(-1.0) + ot: -1.0 + - cd: r.ceil(-1.5) + ot: -1.0 + + - cd: r.expr('X').ceil() + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + + - cd: r.round(1.0).type_of() + ot: "NUMBER" + - cd: r.round(1.0) + ot: 1.0 + - cd: r.expr(1.0).round() + ot: 1.0 + + - cd: r.round(0.5) + ot: 1.0 + - cd: r.round(-0.5) + ot: -1.0 + + - cd: r.round(0.0) + ot: 0.0 + - cd: r.round(1.0) + ot: 1.0 + - cd: r.round(10.0) + ot: 10.0 + - cd: r.round(1000000000.0) + ot: 1000000000.0 + - cd: r.round(1e20) + ot: 1e20 + + - cd: r.round(-1.0) + ot: -1.0 + - cd: r.round(-10.0) + ot: -10.0 + - cd: r.round(-1000000000.0) + ot: -1000000000.0 + - cd: r.round(-1e20) + ot: -1e20 + + - cd: r.round(0.1) + ot: 0.0 + - cd: r.round(1.1) + ot: 1.0 + - cd: r.round(10.1) + ot: 10.0 + - cd: r.round(1000000000.1) + ot: 1000000000.0 + + - cd: r.round(-1.1) + ot: -1.0 + - cd: r.round(-10.1) + ot: -10.0 + - cd: r.round(-1000000000.1) + ot: -1000000000.0 + + - cd: r.round(0.9) + ot: 1.0 + - cd: r.round(9.9) + ot: 10.0 + - cd: r.round(999999999.9) + ot: 1000000000.0 + + - cd: r.round(-0.9) + ot: -1.0 + - cd: r.round(-9.9) + ot: -10.0 + - cd: r.round(-999999999.9) + ot: -1000000000.0 + + - cd: r.expr('X').round() + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/logic.yaml b/ext/librethinkdbxx/test/upstream/math_logic/logic.yaml new file mode 100644 index 00000000..84dbc574 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/logic.yaml @@ -0,0 +1,169 @@ +desc: These tests are aimed at &&, ||, and ! +tests: + + ## basic operator usage + + # Python overloads '&' for 'and' + - py: + - r.expr(true) & true + - true & r.expr(true) + - r.and_(true,true) + - r.expr(true).and_(true) + rb: + - r(true) & true + - r(true) & r(true) + - r.and(true,true) + - r(true).and(true) + js: + - r.and(true,true) + - r(true).and(true) + ot: true + - py: + - r.expr(true) & false + - r.expr(false) & false + - true & r.expr(false) + - false & r.expr(false) + - r.and_(true,false) + - r.and_(false,false) + - r.expr(true).and_(false) + - r.expr(false).and_(false) + rb: + - r(true) & false + - r(false) & false + - r(true) & r(false) + - r(false) & r(false) + - r.and(true,false) + - r.and(false,false) + - r(true).and(false) + - r(false).and(false) + js: + - r.and(true,false) + - r.and(false,false) + - r(true).and(false) + - r(false).and(false) + ot: false + + # Python overloads '|' for 'or' + - py: + - r.expr(true) | true + - r.expr(true) | false + - true | r.expr(true) + - true | r.expr(false) + - r.or_(true,true) + - r.or_(true,false) + - r.expr(true).or_(true) + - r.expr(true).or_(false) + rb: + - r(true) | true + - r(true) | false + - r(true) | r(true) + - r(true) | r(false) + - r.or(true,true) + - r.or(true,false) + - r(true).or(true) + - r(true).or(false) + js: + - r.or(true,true) + - r.or(true,false) + - r(true).or(true) + - r(true).or(false) + ot: true + - py: + - r.expr(false) | false + - false | r.expr(false) + - r.and_(false,false) + - r.expr(false).and_(false) + rb: + - r(false) | false + - r(false) | r(false) + - r.and(false,false) + - r(false).and(false) + js: + - r.and(false,false) + - r(false).and(false) + ot: false + + # Python overloads '~' for 'not' + - py: + - ~r.expr(True) + - r.not_(True) + cd: r(true).not() + ot: false + - py: + - ~r.expr(False) + - r.not_(False) + cd: r(false).not() + ot: true + - py: r.expr(True).not_() + cd: r(true).not() + ot: false + - py: r.expr(False).not_() + cd: r(false).not() + ot: true + + ## DeMorgan's rules! + + - py: + - ~r.and_(True, True) == r.or_(~r.expr(True), ~r.expr(True)) + - ~r.and_(True, False) == r.or_(~r.expr(True), ~r.expr(False)) + - ~r.and_(False, False) == r.or_(~r.expr(False), ~r.expr(False)) + - ~r.and_(False, True) == r.or_(~r.expr(False), ~r.expr(True)) + cd: + - r(true).and(true).not().eq(r(true).not().or(r(true).not())) + - r(true).and(false).not().eq(r(true).not().or(r(false).not())) + - r(false).and(false).not().eq(r(false).not().or(r(false).not())) + - r(false).and(true).not().eq(r(false).not().or(r(true).not())) + ot: true + + # Test multiple arguments to 'and' and 'or' + - cd: r(true).and(true, true, true, true) + py: r.and_(True, True, True, True, True) + ot: true + - cd: r(true).and(true, true, false, true) + py: r.and_(True, True, True, False, True) + ot: false + - cd: r(true).and(false, true, false, true) + py: r.and_(True, False, True, False, True) + ot: false + - cd: r(false).or(false, false, false, false) + py: r.or_(False, False, False, False, False) + ot: false + - cd: r(false).or(false, false, true, false) + py: r.or_(False, False, False, True, False) + ot: true + - cd: r(false).or(true, false, true, false) + py: r.or_(False, True, False, True, False) + ot: true + + # Test that precedence errors are detected + - js: r.expr(r.expr('a')('b')).default(2) + py: r.expr(r.expr('a')['b']).default(2) + rb: r(r('a')['b']).default(2) + ot: err("ReqlQueryLogicError", "Cannot perform bracket on a non-object non-sequence `\"a\"`.", []) + - py: r.expr(r.expr(True) & r.expr(False) == r.expr(False) | r.expr(True)) + ot: err("ReqlDriverCompileError", "Calling '==' on result of infix bitwise operator:", []) + - py: r.expr(r.and_(True, False) == r.or_(False, True)) + ot: False + - rb: r.expr(r.expr(True) & r.expr(False) >= r.expr(False) | r.expr(True)) + py: r.expr(r.expr(True) & r.expr(False) >= r.expr(False) | r.expr(True)) + ot: err("ReqlDriverCompileError", "Calling '>=' on result of infix bitwise operator:", []) + - cd: r.expr(r.and(True, False) >= r.or(False, True)) + py: r.expr(r.and_(True, False) >= r.or_(False, True)) + ot: False + + # Type errors + - py: r.expr(1) & True + cd: r(1).and(true) + ot: true + + - py: r.expr(False) | 'str' + cd: r(false).or('str') + ot: ("str") + + - py: ~r.expr(1) + cd: r(1).not() + ot: false + + - py: ~r.expr(null) + cd: r(null).not() + ot: true diff --git a/ext/librethinkdbxx/test/upstream/math_logic/math.yaml b/ext/librethinkdbxx/test/upstream/math_logic/math.yaml new file mode 100644 index 00000000..5010ac51 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/math.yaml @@ -0,0 +1,11 @@ +desc: Tests of nested arithmetic expressions +tests: + + - py: (((4 + 2 * (r.expr(26) % 18)) / 5) - 3) + js: r(4).add(r(2).mul(r(26).mod(18))).div(5).sub(3) + rb: + - ((((r 4) + (r 2) * ((r 26) % 18)) / 5) -3) + - (((4 + 2 * ((r 26) % 18)) / 5) -3) + ot: 1 + + # Prescedence set by host langauge diff --git a/ext/librethinkdbxx/test/upstream/math_logic/mod.yaml b/ext/librethinkdbxx/test/upstream/math_logic/mod.yaml new file mode 100644 index 00000000..80a0a32c --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/mod.yaml @@ -0,0 +1,34 @@ +desc: Tests for the basic usage of the mod operation +tests: + + - cd: r.expr(10).mod(3) + py: + - r.expr(10) % 3 + - 10 % r.expr(3) + - r.expr(10).mod(3) + rb: + - (r 10) % 3 + - r(10).mod 3 + - 10 % (r 3) + ot: 1 + + - cd: r.expr(-10).mod(-3) + py: r.expr(-10) % -3 + rb: (r -10) % -3 + ot: -1 + + # Type errors + - cd: r.expr(4).mod('a') + py: r.expr(4) % 'a' + rb: r(4) % 'a' + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) + + - cd: r.expr('a').mod(1) + py: r.expr('a') % 1 + rb: r('a') % 1 + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - cd: r.expr('a').mod('b') + py: r.expr('a') % 'b' + rb: r('a') % 'b' + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/mul.yaml b/ext/librethinkdbxx/test/upstream/math_logic/mul.yaml new file mode 100644 index 00000000..ba9b3f9d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/mul.yaml @@ -0,0 +1,60 @@ +desc: Tests for the basic usage of the multiplication operation +tests: + + - cd: r.expr(1).mul(2) + py: + - r.expr(1) * 2 + - 1 * r.expr(2) + - r.expr(1).mul(2) + rb: + - (r 1) * 2 + - r(1).mul(2) + - 1 * (r 2) + ot: 2 + + - py: r.expr(-1) * -1 + js: r(-1).mul(-1) + rb: (r -1) * -1 + ot: 1 + + - cd: r.expr(1.5).mul(4.5) + py: r.expr(1.5) * 4.5 + rb: (r 1.5) * 4.5 + ot: 6.75 + + - py: r.expr([1,2,3]) * 3 + js: r([1,2,3]).mul(3) + rb: (r [1,2,3]) * 3 + ot: [1,2,3,1,2,3,1,2,3] + + - cd: r.expr(1).mul(2,3,4,5) + ot: 120 + + - cd: r(2).mul([1,2,3], 2) + py: # this form does not work in Python + ot: [1,2,3,1,2,3,1,2,3,1,2,3] + + - cd: r([1,2,3]).mul(2, 2) + py: # this form does not work in Python + ot: [1,2,3,1,2,3,1,2,3,1,2,3] + + - cd: r(2).mul(2, [1,2,3]) + py: # this form does not work in Python + ot: [1,2,3,1,2,3,1,2,3,1,2,3] + + # Type errors + - py: r.expr('a') * 0.8 + cd: r('a').mul(0.8) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - py: r.expr(1) * 'a' + cd: r(1).mul('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) + + - py: r.expr('b') * 'a' + cd: r('b').mul('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - py: r.expr([]) * 1.5 + cd: r([]).mul(1.5) + ot: err('ReqlQueryLogicError', 'Number not an integer: 1.5', [0]) diff --git a/ext/librethinkdbxx/test/upstream/math_logic/sub.yaml b/ext/librethinkdbxx/test/upstream/math_logic/sub.yaml new file mode 100644 index 00000000..2c91bdaa --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/math_logic/sub.yaml @@ -0,0 +1,37 @@ +desc: Tests for basic usage of the subtraction operation +tests: + + - cd: r.expr(1).sub(1) + py: + - r.expr(1) - 1 + - 1 - r.expr(1) + - r.expr(1).sub(1) + rb: + - (r 1) - 1 + - 1 - (r 1) + - r(1).sub(1) + - r.expr(1).sub(1) + ot: 0 + + - cd: r.expr(-1).sub(1) + py: r.expr(-1) - 1 + rb: (r -1) - 1 + ot: -2 + + - cd: r.expr(1.75).sub(8.5) + py: r.expr(1.75) - 8.5 + rb: (r 1.75) - 8.5 + ot: -6.75 + + - cd: r.expr(1).sub(2,3,4,5) + ot: -13 + + # Type errors + - cd: r.expr('a').sub(0.8) + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) + + - cd: r.expr(1).sub('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [1]) + + - cd: r.expr('b').sub('a') + ot: err('ReqlQueryLogicError', 'Expected type NUMBER but found STRING.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/meta/composite.py.yaml b/ext/librethinkdbxx/test/upstream/meta/composite.py.yaml new file mode 100644 index 00000000..59a8cf5b --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/meta/composite.py.yaml @@ -0,0 +1,14 @@ +desc: Tests meta operations in composite queries +tests: + + - py: r.expr([1,2,3]).for_each(r.db_create('db_' + r.row.coerce_to('string'))) + ot: ({'dbs_created':3,'config_changes':arrlen(3)}) + + - py: | + r.db_list().set_difference(["rethinkdb", "test"]).for_each(lambda db_name: + r.expr([1,2,3]).for_each(lambda i: + r.db(db_name).table_create('tbl_' + i.coerce_to('string')))) + ot: partial({'tables_created':9}) + + - py: r.db_list().set_difference(["rethinkdb", "test"]).for_each(r.db_drop(r.row)) + ot: partial({'dbs_dropped':3,'tables_dropped':9}) diff --git a/ext/librethinkdbxx/test/upstream/meta/dbs.yaml b/ext/librethinkdbxx/test/upstream/meta/dbs.yaml new file mode 100644 index 00000000..c49460e3 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/meta/dbs.yaml @@ -0,0 +1,51 @@ +desc: Tests meta queries for databases +tests: + + # We should always start out with a 'test' database and the special 'rethinkdb' + # database + - cd: r.db_list() + ot: bag(['rethinkdb', 'test']) + + ## DB create + + - cd: r.db_create('a') + ot: partial({'dbs_created':1}) + - cd: r.db_create('b') + ot: partial({'dbs_created':1}) + + ## DB list + + - cd: r.db_list() + ot: bag(['rethinkdb', 'a', 'b', 'test']) + + ## DB config + + - cd: r.db('a').config() + ot: {'name':'a','id':uuid()} + + ## DB drop + + - cd: r.db_drop('b') + ot: partial({'dbs_dropped':1}) + + - cd: r.db_list() + ot: bag(['rethinkdb', 'a', 'test']) + + - cd: r.db_drop('a') + ot: partial({'dbs_dropped':1}) + + - cd: r.db_list() + ot: bag(['rethinkdb', 'test']) + + ## DB errors + - cd: r.db_create('bar') + ot: partial({'dbs_created':1}) + + - cd: r.db_create('bar') + ot: err('ReqlOpFailedError', 'Database `bar` already exists.', [0]) + + - cd: r.db_drop('bar') + ot: partial({'dbs_dropped':1}) + + - cd: r.db_drop('bar') + ot: err('ReqlOpFailedError', 'Database `bar` does not exist.', [0]) diff --git a/ext/librethinkdbxx/test/upstream/meta/table.yaml b/ext/librethinkdbxx/test/upstream/meta/table.yaml new file mode 100644 index 00000000..940a9d27 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/meta/table.yaml @@ -0,0 +1,365 @@ +desc: Tests meta queries for creating and deleting tables +tests: + + - def: db = r.db('test') + + - cd: db.table_list() + ot: [] + + - cd: r.db('rethinkdb').info() + ot: ({'type':'DB','name':'rethinkdb','id':null}) + + - cd: r.db('rethinkdb').table('stats').info() + ot: partial({'db':{'type':'DB','name':'rethinkdb','id':null}, + 'type':'TABLE','id':null,'name':'stats', + 'indexes':[],'primary_key':'id'}) + + # Table create + - cd: db.table_create('a') + ot: partial({'tables_created':1}) + + - cd: db.table_list() + ot: ['a'] + + - cd: db.table_create('b') + ot: partial({'tables_created':1}) + + - cd: db.table_list() + ot: bag(['a', 'b']) + + # Table drop + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + + - cd: db.table_list() + ot: ['b'] + + - cd: db.table_drop('b') + ot: partial({'tables_dropped':1}) + + - cd: db.table_list() + ot: [] + + # Table create options + - py: db.table_create('ab', durability='soft') + js: db.table_create('ab', {durability:'soft'}) + rb: db.table_create('ab', :durability => 'soft') + ot: partial({'tables_created':1,'config_changes':[partial({'new_val':partial({'durability':'soft'})})]}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', durability='hard') + js: db.table_create('ab', {durability:'hard'}) + rb: db.table_create('ab', :durability => 'hard') + ot: partial({'tables_created':1,'config_changes':[partial({'new_val':partial({'durability':'hard'})})]}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', durability='fake') + js: db.table_create('ab', {durability:'fake'}) + rb: db.table_create('ab', :durability => 'fake') + ot: err('ReqlQueryLogicError', 'Durability option `fake` unrecognized (options are "hard" and "soft").') + + - py: db.table_create('ab', primary_key='bar', shards=2, replicas=1) + js: db.tableCreate('ab', {primary_key:'bar', shards:2, replicas:1}) + rb: db.table_create('ab', {:primary_key => 'bar', :shards => 1, :replicas => 1}) + ot: partial({'tables_created':1}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', primary_key='bar', primary_replica_tag='default') + js: db.tableCreate('ab', {primary_key:'bar', primaryReplicaTag:'default'}) + rb: db.table_create('ab', {:primary_key => 'bar', :primary_replica_tag => 'default'}) + ot: partial({'tables_created':1}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + - py: db.table_create('ab', nonvoting_replica_tags=['default']) + js: db.tableCreate('ab', {nonvotingReplicaTags:['default']}) + rb: db.table_create('ab', {:nonvoting_replica_tags => ['default']}) + ot: partial({'tables_created':1}) + + - cd: db.table_drop('ab') + ot: partial({'tables_dropped':1}) + + # Table reconfigure + - cd: db.table_create('a') + ot: partial({'tables_created':1}) + + - py: db.table('a').reconfigure(shards=1, replicas=1) + js: db.table('a').reconfigure({shards:1, replicas:1}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 1) + ot: partial({'reconfigured':1}) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":1}, nonvoting_replica_tags=['default'], primary_replica_tag='default') + js: db.table('a').reconfigure({shards:1, replicas:{default:1}, nonvoting_replica_tags:['default'], primary_replica_tag:'default'}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 1}, :nonvoting_replica_tags => ['default'], :primary_replica_tag => 'default') + ot: partial({'reconfigured':1}) + + - py: db.table('a').reconfigure(shards=1, replicas=1, dry_run=True) + js: db.table('a').reconfigure({shards:1, replicas:1, dry_run:true}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 1, :dry_run => true) + ot: partial({'reconfigured':0}) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback") + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback"}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback") + ot: err('ReqlOpFailedError', 'This table doesn\'t need to be repaired.', []) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback", dry_run=True) + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback", dry_run:true}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback", :dry_run => true) + ot: err('ReqlOpFailedError', 'This table doesn\'t need to be repaired.', []) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback_or_erase") + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback_or_erase"}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback_or_erase") + ot: err('ReqlOpFailedError', 'This table doesn\'t need to be repaired.', []) + + - py: db.table('a').reconfigure(emergency_repair=None, shards=1, replicas=1, dry_run=True) + js: db.table('a').reconfigure({emergency_repair:null, shards:1, replicas:1, dry_run:true}) + rb: db.table('a').reconfigure(:emergency_repair => null, :shards => 1, :replicas => 1, :dry_run => true) + ot: partial({'reconfigured':0}) + + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + + # Table errors + - cd: db.table_create('foo') + ot: partial({'tables_created':1}) + + - cd: db.table_create('foo') + ot: err('ReqlOpFailedError', 'Table `test.foo` already exists.', [0]) + + - cd: db.table_drop('foo') + ot: partial({'tables_dropped':1}) + + - cd: db.table_drop('foo') + ot: err('ReqlOpFailedError', 'Table `test.foo` does not exist.', [0]) + + - cd: db.table_create('nonsense', 'foo') + ot: + js: err('ReqlCompileError', 'Expected 1 argument (not including options) but found 2.', []) + rb: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + py: err("ReqlCompileError", "Expected between 1 and 2 arguments but found 3.", []) + + - js: db.table_create('nonsense', {'foo':'bar'}) + py: db.table_create('nonsense', foo='bar') + rb: db.table_create('nonsense', :foo => 'bar') + ot: err('ReqlCompileError', "Unrecognized optional argument `foo`.", []) + + # RSI(reql_admin): Add tests for table_create() with configuration parameters + + # Table reconfigure errors + - cd: db.table_create('a') + ot: partial({'tables_created':1}) + + - py: db.table('a').reconfigure(shards=0, replicas=1) + js: db.table('a').reconfigure({shards:0, replicas:1}) + rb: db.table('a').reconfigure(:shards => 0, :replicas => 1) + ot: err('ReqlQueryLogicError', 'Every table must have at least one shard.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":1}, primary_replica_tag="foo") + js: db.table('a').reconfigure({shards:1, replicas:{default:1}, primary_replica_tag:"foo"}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 1}, :primary_replica_tag => "foo") + ot: err('ReqlOpFailedError', 'Can\'t use server tag `foo` for primary replicas because you specified no replicas in server tag `foo`.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":1}, primary_replica_tag="default", nonvoting_replica_tags=["foo"]) + js: db.table('a').reconfigure({shards:1, replicas:{"default":1}, primary_replica_tag:"default", nonvoting_replica_tags:["foo"]}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 1}, :primary_replica_tag => "default", :nonvoting_replica_tags => ["foo"]) + ot: err('ReqlOpFailedError', 'You specified that the replicas in server tag `foo` should be non-voting, but you didn\'t specify a number of replicas in server tag `foo`.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"foo":0}, primary_replica_tag="foo") + js: db.table('a').reconfigure({shards:1, replicas:{foo:0}, primary_replica_tag:"foo"}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:foo => 0}, :primary_replica_tag => "foo") + ot: err('ReqlOpFailedError', 'You must set `replicas` to at least one. `replicas` includes the primary replica; if there are zero replicas, there is nowhere to put the data.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":0}) + js: db.table('a').reconfigure({shards:1, replicas:{default:0}}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => 0}) + ot: err('ReqlQueryLogicError', '`primary_replica_tag` must be specified when `replicas` is an OBJECT.', []) + + - py: db.table('a').reconfigure(shards=1, replicas={"default":-3}, primary_replica_tag='default') + js: db.table('a').reconfigure({shards:1, replicas:{default:-3}, primary_replica_tag:'default'}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => {:default => -3}, :primary_replica_tag => 'default') + ot: err('ReqlQueryLogicError', 'Can\'t have a negative number of replicas', []) + + - py: db.table('a').reconfigure(shards=1, replicas=3, primary_replica_tag='foo') + js: db.table('a').reconfigure({shards:1, replicas:3, primary_replica_tag:'foo'}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 3, :primary_replica_tag => 'foo') + ot: err('ReqlQueryLogicError', '`replicas` must be an OBJECT if `primary_replica_tag` is specified.', []) + + - py: db.table('a').reconfigure(shards=1, replicas=3, nonvoting_replica_tags=['foo']) + js: db.table('a').reconfigure({shards:1, replicas:3, nonvoting_replica_tags:['foo']}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 3, :nonvoting_replica_tags => ['foo']) + ot: err('ReqlQueryLogicError', '`replicas` must be an OBJECT if `nonvoting_replica_tags` is specified.', []) + + - py: db.reconfigure(emergency_repair="unsafe_rollback") + js: db.reconfigure({emergency_repair:"unsafe_rollback"}) + rb: db.reconfigure(:emergency_repair => "unsafe_rollback") + ot: err('ReqlQueryLogicError', 'Can\'t emergency repair an entire database at once; instead you should run `reconfigure()` on each table individually.') + + - py: db.table('a').reconfigure(emergency_repair="foo") + js: db.table('a').reconfigure({emergency_repair:"foo"}) + rb: db.table('a').reconfigure(:emergency_repair => "foo") + ot: err('ReqlQueryLogicError', '`emergency_repair` should be "unsafe_rollback" or "unsafe_rollback_or_erase"', []) + + - py: db.table('a').reconfigure(emergency_repair="unsafe_rollback", shards=1, replicas=1) + js: db.table('a').reconfigure({emergency_repair:"unsafe_rollback", shards:1, replicas:1}) + rb: db.table('a').reconfigure(:emergency_repair => "unsafe_rollback", :shards => 1, :replicas => 1) + ot: err('ReqlQueryLogicError', 'In emergency repair mode, you can\'t specify shards, replicas, etc.') + + # Test reconfigure auto-sharding without data + - py: db.table('a').reconfigure(shards=2, replicas=1) + js: db.table('a').reconfigure({shards:2, replicas:1}) + rb: db.table('a').reconfigure(:shards => 2, :replicas => 1) + ot: partial({'reconfigured':1}) + + - py: db.table('a').wait(wait_for="all_replicas_ready") + js: db.table('a').wait({"waitFor":"all_replicas_ready"}) + rb: db.table('a').wait(:wait_for=>"all_replicas_ready") + ot: {"ready":1} + + # Insert some data so that `reconfigure()` can pick shard points + - py: db.table('a').insert([{"id":1}, {"id":2}, {"id":3}, {"id":4}]) + js: db.table('a').insert([{id:1}, {id:2}, {id:3}, {id:4}]) + rb: db.table('a').insert([{"id" => 1}, {"id" => 2}, {"id" => 3}, {"id" => 4}]) + ot: partial({"inserted":4}) + + - py: db.table('a').reconfigure(shards=2, replicas=1) + js: db.table('a').reconfigure({shards:2, replicas:1}) + rb: db.table('a').reconfigure(:shards => 2, :replicas => 1) + ot: partial({'reconfigured':1}) + + - py: db.table('a').reconfigure(shards=1, replicas=2) + js: db.table('a').reconfigure({shards:1, replicas:2}) + rb: db.table('a').reconfigure(:shards => 1, :replicas => 2) + ot: err('ReqlOpFailedError', 'Can\'t put 2 replicas on servers with the tag `default` because there are only 1 servers with the tag `default`. It\'s impossible to have more replicas of the data than there are servers.', []) + + # Test wait and rebalance + - py: db.table('a').wait(wait_for="all_replicas_ready") + js: db.table('a').wait({"waitFor":"all_replicas_ready"}) + rb: db.table('a').wait(:wait_for=>"all_replicas_ready") + ot: {"ready":1} + - cd: db.table('a').rebalance() + ot: partial({'rebalanced':1}) + + - py: db.wait(wait_for="all_replicas_ready") + js: db.wait({"waitFor":"all_replicas_ready"}) + rb: db.wait(:wait_for=>"all_replicas_ready") + ot: {"ready":1} + - cd: db.rebalance() + ot: partial({'rebalanced':1}) + + - cd: r.wait() + ot: + py: err('AttributeError', "'module' object has no attribute 'wait'", []) + # different sub-versions of node have different messages #5617 + js: err('TypeError') + rb: err('ReqlQueryLogicError', '`wait` can only be called on a table or database.', []) + - cd: r.rebalance() + ot: + py: err('AttributeError', "'module' object has no attribute 'rebalance'", []) + # different sub-versions of node have different messages #5617 + js: err('TypeError') + rb: err('ReqlQueryLogicError', '`rebalance` can only be called on a table or database.', []) + + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + + # Reconfiguring all tables in a database + - cd: db.table_create('a') + - cd: db.table_create('b') + - cd: db.table_create('c') + + - py: db.reconfigure(shards=0, replicas=1) + js: db.reconfigure({shards:0, replicas:1}) + rb: db.reconfigure(:shards => 0, :replicas => 1) + ot: err('ReqlQueryLogicError', 'Every table must have at least one shard.', []) + + - py: db.reconfigure(shards=1, replicas={"default":0}) + js: db.reconfigure({shards:1, replicas:{default:0}}) + rb: db.reconfigure(:shards => 1, :replicas => {:default => 0}) + ot: err('ReqlQueryLogicError', '`primary_replica_tag` must be specified when `replicas` is an OBJECT.', []) + + - py: db.reconfigure(shards=1, replicas={"default":-3}, primary_replica_tag='default') + js: db.reconfigure({shards:1, replicas:{default:-3}, primary_replica_tag:'default'}) + rb: db.reconfigure(:shards => 1, :replicas => {:default => -3}, :primary_replica_tag => 'default') + ot: err('ReqlQueryLogicError', 'Can\'t have a negative number of replicas', []) + + - py: db.reconfigure(shards=1, replicas=3, primary_replica_tag='foo') + js: db.reconfigure({shards:1, replicas:3, primary_replica_tag:'foo'}) + rb: db.reconfigure(:shards => 1, :replicas => 3, :primary_replica_tag => 'foo') + ot: err('ReqlQueryLogicError', '`replicas` must be an OBJECT if `primary_replica_tag` is specified.', []) + + - py: db.reconfigure(shards=2, replicas=1) + js: db.reconfigure({shards:2, replicas:1}) + rb: db.reconfigure(:shards => 2, :replicas => 1) + ot: partial({'reconfigured':3}) + + - cd: db.table_drop('a') + ot: partial({'tables_dropped':1}) + - cd: db.table_drop('b') + ot: partial({'tables_dropped':1}) + - cd: db.table_drop('c') + ot: partial({'tables_dropped':1}) + + # table_config and table_status porcelains + - cd: r.db_create("test2") + ot: partial({'dbs_created':1}) + + - def: db2 = r.db("test2") + + - cd: db.table_create("testA") + ot: partial({'tables_created':1}) + - cd: db.table_create("testB") + ot: partial({'tables_created':1}) + - cd: db2.table_create("test2B") + ot: partial({'tables_created':1}) + + - cd: r.table('testA').config().pluck('db','name') + ot: {'db':'test','name':'testA'} + + - cd: r.table('doesntexist').config() + ot: err('ReqlOpFailedError', 'Table `test.doesntexist` does not exist.', []) + + - cd: r.table('test2B').config() + ot: err('ReqlOpFailedError', 'Table `test.test2B` does not exist.', []) + + - cd: r.db('rethinkdb').table('table_config').filter({'name':'testA'}).nth(0).eq(r.table('testA').config()) + ot: True + + - cd: r.db('rethinkdb').table('table_status').filter({'name':'testA'}).nth(0).eq(r.table('testA').status()) + ot: True + + - py: r.db('rethinkdb').table('table_config', identifier_format='uuid').nth(0)["db"] + js: r.db('rethinkdb').table('table_config', {identifierFormat:'uuid'}).nth(0)("db") + rb: r.db('rethinkdb').table('table_config', {:identifier_format=>'uuid'}).nth(0)["db"] + ot: uuid() + + - py: r.table('testA', identifier_format='uuid').count() + js: r.table('testA', {identifierFormat:'uuid'}).count() + rb: r.table('testA', {:identifier_format=>'uuid'}).count() + ot: 0 + + - py: r.wait(wait_for='all_replicas_ready', timeout=5) + js: r.wait({waitFor:'all_replicas_ready', timeout:5}) + rb: r.wait(:wait_for=>'all_replicas_ready', :timeout => 5) + ot: + py: err('AttributeError', "'module' object has no attribute 'wait'", []) + # different sub-versions of node have different messages #5617 + js: err('TypeError') + rb: err('ReqlQueryLogicError', '`wait` can only be called on a table or database.', []) + + - cd: db.table_drop('testA') + ot: partial({'tables_dropped':1}) + + - cd: db.table_drop('testB') + ot: partial({'tables_dropped':1}) + + - cd: r.db_drop('test2') + ot: partial({'dbs_dropped':1,'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml b/ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml new file mode 100644 index 00000000..ef1621a7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/atomic_get_set.yaml @@ -0,0 +1,93 @@ +desc: Tests replacement of selections +table_variable_name: tbl +tests: + + # old version of argument + - cd: tbl.insert({'id':0}, :return_vals => true).pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_vals=True).pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return_vals':true}).pluck('changes', 'first__error') + ot: err("ReqlQueryLogicError", "Error:"+" encountered obsolete optarg `return_vals`. Use `return_changes` instead.", [0]) + + - cd: tbl.insert({'id':0}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_changes=True).pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':null,'new_val':{'id':0}}]}) + - cd: tbl.insert({'id':0}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_changes=True).pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[], 'first_error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}"}) + - cd: tbl.insert({'id':0}, return_changes:'always').pluck('changes', 'first_error') + py: tbl.insert({'id':0}, return_changes='always').pluck('changes', 'first_error') + js: tbl.insert({'id':0}, {'return__changes':'always'}).pluck('changes', 'first__error') + ot: ({'first_error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}",'changes':[{'old_val':{'id':0},'new_val':{'id':0},'error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}"}]}) + - cd: tbl.insert([{'id':1}], :return_changes => true) + py: tbl.insert([{'id':1}], return_changes=True) + js: tbl.insert([{'id':1}], {'return__changes':true}) + ot: ({'changes':[{'new_val':{'id':1},'old_val':null}], 'errors':0, 'deleted':0, 'unchanged':0, 'skipped':0, 'replaced':0, 'inserted':1}) + - cd: tbl.insert([{'id':0}], :return_changes => true).pluck('changes', 'first_error') + py: tbl.insert([{'id':0}], return_changes=True).pluck('changes', 'first_error') + js: tbl.insert([{'id':0}], {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[],'first_error':"Duplicate primary key `id`:\n{\n\t\"id\":\t0\n}\n{\n\t\"id\":\t0\n}"}) + + - cd: tbl.get(0).update({'x':1}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).update({'x':1}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).update({'x':1}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':{'id':0},'new_val':{'id':0,'x':1}}]}) + - cd: tbl.get(0).update({'x':r.error("a")}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).update({'x':r.error("a")}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).update({'x':r.error("a")}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[],'first_error':'a'}) + - rb: tbl.update({'x':3}, :return_changes => true).pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + py: tbl.update({'x':3}, return_changes=True).pluck('changes', 'first_error').do(lambda d:d.merge({'changes':d['changes'].order_by(lambda a:a['old_val']['id'])})) + js: tbl.update({'x':3}, {'return__changes':true}).pluck('changes', 'first__error').do(function(p){return p.merge({'changes':p('changes').orderBy(function(a){return a('old__val')('id')})})}) + ot: ({'changes':[{'old_val':{'id':0, 'x':1},'new_val':{'id':0, 'x':3}}, {'old_val':{'id':1},'new_val':{'id':1, 'x':3}}]}) + + - cd: tbl.get(0).replace({'id':0,'x':2}, :return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).replace({'id':0,'x':2}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).replace({'id':0,'x':2}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':{'id':0,'x':3},'new_val':{'id':0,'x':2}}]}) + - cd: tbl.get(0).replace(:return_changes => true){{'x':r.error('a')}}.pluck('changes', 'first_error') + py: tbl.get(0).replace(lambda y:{'x':r.error('a')}, return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).replace(function(y){return {'x':r.error('a')}}, {'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[],'first_error':'a'}) + - cd: tbl.get(0).replace(:return_changes => 'always'){{'x':r.error('a')}}.pluck('changes', 'first_error') + py: tbl.get(0).replace(lambda y:{'x':r.error('a')}, return_changes='always').pluck('changes', 'first_error') + js: tbl.get(0).replace(function(y){return {'x':r.error('a')}}, {'return__changes':'always'}).pluck('changes', 'first__error') + ot: ({'first_error':'a','changes':[{'old_val':{'id':0,'x':2},'new_val':{'id':0,'x':2},'error':'a'}]}) + - rb: tbl.replace( :return_changes => true) { |d| d.without('x')}.pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + py: tbl.replace(lambda y:y.without('x'), return_changes=True).pluck('changes', 'first_error').do(lambda d:d.merge({'changes':d['changes'].order_by(lambda a:a['old_val']['id'])})) + js: tbl.replace(function(p){return p.without('x')}, {'return__changes':true}).pluck('changes', 'first__error').do(function(p){return p.merge({'changes':p('changes').orderBy(function(a){return a('old__val')('id')})})}) + ot: ({'changes':[{'new_val':{'id':0},'old_val':{'id':0, 'x':2}}, {'new_val':{'id':1},'old_val':{'id':1,'x':3}}]}) + - rb: tbl.replace({'x':1}, :return_changes => 'always').pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + py: tbl.replace({'x':1}, return_changes='always').pluck('changes', 'first_error').do(lambda d:d.merge({'changes':d['changes'].order_by(lambda a:a['old_val']['id'])})) + js: tbl.replace({'x':1}, {'return__changes':'always'}).pluck('changes', 'first__error').do(function(p){return p.merge({'changes':p('changes').orderBy(function(a){return a('old__val')('id')})})}) + ot: ({'first_error':"Inserted object must have primary key `id`:\n{\n\t\"x\":\t1\n}", 'changes':[{'new_val':{'id':0},'old_val':{'id':0}, 'error':"Inserted object must have primary key `id`:\n{\n\t\"x\":\t1\n}"}, {'new_val':{'id':1},'old_val':{'id':1},'error':"Inserted object must have primary key `id`:\n{\n\t\"x\":\t1\n}"}]}) + + - rb: tbl.foreach{|row| [tbl.get(0).update(null, :return_changes => true), tbl.get(0).update({a:1}, :return_changes => true)]}.pluck('changes', 'first_error').do {|d| d.merge({:changes => d['changes'].order_by {|a| a['old_val']['id']}})} + ot: ({'changes':[{"new_val"=>{"a"=>1, "id"=>0}, "old_val"=>{"id"=>0}}]}) + - rb: tbl.get(0).update({a:r.literal()})['replaced'] + ot: 1 + + - rb: tbl.get(0).update({}).pluck('changes') + ot: ({}) + - rb: tbl.get(0).update({}, return_changes:true).pluck('changes') + ot: ({'changes':[]}) + - rb: tbl.get(0).update({}, return_changes:'always').pluck('changes') + ot: ({'changes':[{'new_val':{'id':0}, 'old_val':{'id':0}}]}) + + - rb: tbl.get(-1).update({}).pluck('changes') + ot: ({}) + - rb: tbl.get(-1).update({}, return_changes:true).pluck('changes') + ot: ({'changes':[]}) + - rb: tbl.get(-1).update({}, return_changes:'always').pluck('changes') + ot: ({'changes':[{'new_val':null, 'old_val':null}]}) + + - cd: tbl.get(0).delete(:return_changes => true).pluck('changes', 'first_error') + py: tbl.get(0).delete(return_changes=True).pluck('changes', 'first_error') + js: tbl.get(0).delete({'return__changes':true}).pluck('changes', 'first__error') + ot: ({'changes':[{'old_val':{'id':0},'new_val':null}]}) + - cd: tbl.delete(:return_changes => true) + py: tbl.delete(return_changes=True) + js: tbl.delete({'return__changes':true}) + ot: ({'deleted':1,'errors':0,'inserted':0,'replaced':0,'skipped':0,'unchanged':0,'changes':[{'new_val':null, 'old_val':{'id':1}}]}) + diff --git a/ext/librethinkdbxx/test/upstream/mutation/delete.yaml b/ext/librethinkdbxx/test/upstream/mutation/delete.yaml new file mode 100644 index 00000000..b222dba9 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/delete.yaml @@ -0,0 +1,50 @@ +desc: Tests deletes of selections +table_variable_name: tbl +tests: + + # Set up some data + + - py: tbl.insert([{'id':i} for i in xrange(100)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id: i}); + } + return res; + }()) + rb: tbl.insert((1..100).map{ |i| {"id" => i} }) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':100}) + + - cd: tbl.count() + ot: 100 + + # Point delete + + - cd: tbl.get(12).delete() + ot: ({'deleted':1,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0}) + + # Attempt deletion with bad durability flag. + + - js: tbl.skip(50).delete({durability:'wrong'}) + rb: tbl.skip(50).delete({ :durability => 'wrong' }) + py: tbl.skip(50).delete(durability='wrong') + ot: err('ReqlQueryLogicError', 'Durability option `wrong` unrecognized (options are "hard" and "soft").', [0]) + + # Delete selection of table, soft durability flag. + + - js: tbl.skip(50).delete({durability:'soft'}) + rb: tbl.skip(50).delete({ :durability => 'soft' }) + py: tbl.skip(50).delete(durability='soft') + ot: ({'deleted':49,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0}) + + # Delete whole table, hard durability flag. + + - js: tbl.delete({durability:'hard'}) + rb: tbl.delete({ :durability => 'hard' }) + py: tbl.delete(durability='hard') + ot: ({'deleted':50,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0}) + + # test deletion on a non-deletable object + - cd: r.expr([1, 2]).delete() + ot: err('ReqlQueryLogicError', 'Expected type SELECTION but found DATUM:', [0]) diff --git a/ext/librethinkdbxx/test/upstream/mutation/insert.yaml b/ext/librethinkdbxx/test/upstream/mutation/insert.yaml new file mode 100644 index 00000000..623a6edc --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/insert.yaml @@ -0,0 +1,239 @@ +desc: Tests insertion into tables +table_variable_name: tbl +tests: + + # Set up our secondary test table + - cd: r.db('test').table_create('test2') + ot: partial({'tables_created':1}) + + - def: tbl2 = r.db('test').table('test2') + + # Single doc insert + - cd: tbl.insert({'id':0,'a':0}) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + - cd: tbl.count() + ot: 1 + + # Hard durability insert + - py: tbl.insert({'id':1, 'a':1}, durability='hard') + js: tbl.insert({id:1, a:1}, {durability:'hard'}) + rb: tbl.insert({ :id => 1, :a => 1 }, { :durability => 'hard' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + - cd: tbl.count() + ot: 2 + + # Soft durability insert + - py: tbl.insert({'id':2, 'a':2}, durability='soft') + js: tbl.insert({id:2, a:2}, {durability:'soft'}) + rb: tbl.insert({ :id => 2, :a => 2 }, { :durability => 'soft' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + - cd: tbl.count() + ot: 3 + + # Wrong durability insert + - py: tbl.insert({'id':3, 'a':3}, durability='wrong') + js: tbl.insert({id:3, a:3}, {durability:'wrong'}) + rb: tbl.insert({ :id => 3, :a => 3 }, { :durability => 'wrong' }) + ot: err('ReqlQueryLogicError', 'Durability option `wrong` unrecognized (options are "hard" and "soft").', [0]) + - cd: tbl.count() + ot: 3 + + # Cleanup. + - cd: tbl.get(2).delete() + ot: {'deleted':1,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':0} + + # Multi doc insert + - cd: tbl.insert([{'id':2,'a':2}, {'id':3,'a':3}]) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':2} + + # Stream insert + - cd: tbl2.insert(tbl) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':4} + + # test pkey clash error + - cd: tbl.insert({'id':2,'b':20}) + ot: {'first_error':"Duplicate primary key `id`:\n{\n\t\"a\":\t2,\n\t\"id\":\t2\n}\n{\n\t\"b\":\t20,\n\t\"id\":\t2\n}",'deleted':0,'replaced':0,'unchanged':0,'errors':1,'skipped':0,'inserted':0} + + # test error conflict option (object exists) + - py: tbl.insert({'id':2,'b':20}, conflict='error') + js: tbl.insert({'id':2,'b':20}, {conflict:'error'}) + rb: tbl.insert({:id => 2, :b => 20}, { :conflict => 'error' }) + ot: {'first_error':"Duplicate primary key `id`:\n{\n\t\"a\":\t2,\n\t\"id\":\t2\n}\n{\n\t\"b\":\t20,\n\t\"id\":\t2\n}",'deleted':0,'replaced':0,'unchanged':0,'errors':1,'skipped':0,'inserted':0} + + # test error conflict option (object doesn't exist) + - py: tbl.insert({'id':15,'b':20}, conflict='error') + js: tbl.insert({'id':15,'b':20}, {conflict:'error'}) + rb: tbl.insert({:id => 15, :b => 20}, { :conflict => 'error' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tbl.get(15) + ot: {'id':15,'b':20} + + # test replace conflict option (object exists) + - py: tbl.insert({'id':2,'b':20}, conflict='replace') + js: tbl.insert({'id':2,'b':20}, {conflict:'replace'}) + rb: tbl.insert({:id => 2, :b => 20}, { :conflict => 'replace' }) + ot: {'deleted':0,'replaced':1,'unchanged':0,'errors':0,'skipped':0,'inserted':0} + + - cd: tbl.get(2) + ot: {'id':2,'b':20} + + # test replace conflict option (object doesn't exist) + - py: tbl.insert({'id':20,'b':20}, conflict='replace') + js: tbl.insert({'id':20,'b':20}, {conflict:'replace'}) + rb: tbl.insert({:id => 20, :b => 20}, { :conflict => 'replace' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tbl.get(20) + ot: {'id':20,'b':20} + + # test update conflict option (object exists) + - py: tbl.insert({'id':2,'c':30}, conflict='update') + js: tbl.insert({'id':2,'c':30}, {conflict:'update'}) + rb: tbl.insert({:id => 2, :c => 30}, { :conflict => 'update' }) + ot: {'deleted':0,'replaced':1,'unchanged':0,'errors':0,'skipped':0,'inserted':0} + + - cd: tbl.get(2) + ot: {'id':2, 'b':20, 'c':30} + + # test update conflict option (object doesn't exist) + - py: tbl.insert({'id':30,'b':20}, conflict='update') + js: tbl.insert({'id':30,'b':20}, {conflict:'update'}) + rb: tbl.insert({:id => 30, :b => 20}, { :conflict => 'update' }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tbl.get(30) + ot: {'id':30,'b':20} + + # test incorrect conflict option + - py: tbl.insert({'id':3, 'a':3}, conflict='wrong') + js: tbl.insert({id:3, a:3}, {conflict:'wrong'}) + rb: tbl.insert({ :id => 3, :a => 3 }, { :conflict => 'wrong' }) + ot: err('ReqlQueryLogicError', 'Conflict option `wrong` unrecognized (options are "error", "replace" and "update").', [0]) + + # test auto pkey generation + - py: r.db('test').table_create('testpkey', primary_key='foo') + js: r.db('test').tableCreate('testpkey', {primaryKey:'foo'}) + rb: r.db('test').table_create('testpkey', { :primary_key => 'foo' }) + ot: partial({'tables_created':1}) + + def: tblpkey = r.db('test').table('testpkey') + + - cd: tblpkey.insert({}) + ot: {'deleted':0,'replaced':0,'generated_keys':arrlen(1,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: tblpkey + ot: [{'foo':uuid()}] + + # test replace conflict pkey generation + - py: tblpkey.insert({'b':20}, conflict='replace') + js: tblpkey.insert({'b':20}, {conflict:'replace'}) + rb: tblpkey.insert({:b => 20}, { :conflict => 'replace' }) + ot: {'deleted':0,'replaced':0,'generated_keys':arrlen(1,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + # test update conflict pkey generation + - py: tblpkey.insert({'b':20}, conflict='update') + js: tblpkey.insert({'b':20}, {conflict:'update'}) + rb: tblpkey.insert({:b => 20}, { :conflict => 'update' }) + ot: {'deleted':0,'replaced':0,'generated_keys':arrlen(1,uuid()),'unchanged':0,'errors':0,'skipped':0,'inserted':1} + + - cd: r.db('test').table_drop('testpkey') + ot: partial({'tables_dropped':1}) + + # Insert within for each + - py: tbl.for_each(lambda row: tbl2.insert(row.merge({'id':row['id'] + 100 })) ) + js: tbl.forEach(function(row) { return tbl2.insert(row.merge({'id':row('id').add(100)})); }) + rb: tbl.for_each(proc { |row| tbl2.insert(row.merge({'id'=>row['id'] + 100 })) }) + ot: {'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':7} + + # Insert unwritable data + - cd: tbl.insert({'value':r.minval}) + rb: tbl.insert({:value => r.minval}) + ot: partial({'errors':1,'first_error':'`r.minval` and `r.maxval` cannot be written to disk.'}) + + - cd: tbl.insert({'value':r.maxval}) + rb: tbl.insert({:value => r.maxval}) + ot: partial({'errors':1,'first_error':'`r.minval` and `r.maxval` cannot be written to disk.'}) + + # Crash 5683 + - py: tbl.insert([{'id':666}, {'id':666}], return_changes="always") + ot: {'changes': [{'new_val': {'id': 666}, 'old_val': None},{'error': 'Duplicate primary key `id`:\n{\n\t"id":\t666\n}\n{\n\t"id":\t666\n}','new_val': {'id': 666},'old_val': {'id': 666}}],'deleted': 0,'errors': 1,'first_error': 'Duplicate primary key `id`:\n{\n\t"id":\t666\n}\n{\n\t"id":\t666\n}','inserted': 1,'replaced': 0,'skipped': 0,'unchanged': 0} + + # Confirm inserts are ordered in return_changes always + - py: tbl.insert([{'id':100+i, 'ordered-num':i} for i in range(1,100)], return_changes="always") + ot: partial({'changes':[{'old_val': None, 'new_val': {'id': 100+i, 'ordered-num': i}} for i in range(1,100)] }) + + # Confirm inserts are ordered in return_changes always with complicated key + - py: tbl.insert([{'id':[1, "blah", 200+i], 'ordered-num':i} for i in range(1,100)], return_changes="always") + ot: partial({'changes':[{'old_val': None, 'new_val': {'id': [1,"blah", 200+i], 'ordered-num': i}} for i in range(1,100)] }) + + # Confirm inserts are ordered in return_changes always with return_changes=true + - py: tbl.insert([{'id':[1, "blah", 300+i], 'ordered-num':i} for i in range(1,100)], return_changes=true) + ot: partial({'changes':[{'old_val': None, 'new_val': {'id': [1,"blah", 300+i], 'ordered-num': i}} for i in range(1,100)] }) + + # Confirm errors are property returned with return_changes="always" + - py: tbl.insert([{'id':100 + i, 'ordered-num':i} for i in range(1,100)], return_changes="always") + ot: partial({'changes':[{'old_val': {'id':100+i, 'ordered-num':i}, 'new_val': {'id':100+i, 'ordered-num':i}, 'error':'Duplicate primary key `id`:\n{\n\t"id":\t'+str(100+i)+',\n\t"ordered-num":\t'+str(i)+'\n}\n{\n\t"id":\t'+str(100+i)+',\n\t"ordered-num":\t'+str(i)+'\n}'} for i in range(1,100)]}) + + # Trivial errors with return_changes="always", this has to be long test, testing order and message for this type + - py: tbl.insert([{'id':123}, {'id':'a'*500}, {'id':321}], return_changes="always") + ot: {'changes': [{'error': 'Duplicate primary key `id`:\n{\n\t"id":\t123,\n\t"ordered-num":\t23\n}\n{\n\t"id":\t123\n}', 'new_val': {'id': 123, 'ordered-num': 23}, 'old_val': {'id': 123, 'ordered-num': 23}}, {'error': 'Primary key too long (max 127 characters): "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"', 'new_val': None, 'old_val': None}, {'new_val': {'id': 321}, 'old_val': None}], 'deleted': 0, 'errors': 2, 'first_error': 'Primary key too long (max 127 characters): "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"', 'inserted': 1, 'replaced': 0, 'skipped': 0, 'unchanged': 0} + + # No errors returned with return_changes=true + - py: tbl.insert([{'id':100 + i, 'ordered-num':i} for i in range(1,100)], return_changes=true) + ot: partial({'changes':[]}) + + - py: tbl.insert({'a':r.minval}, return_changes="always") + ot: partial({'changes': [{'old_val': None, 'new_val': None, 'error': '`r.minval` and `r.maxval` cannot be written to disk.'}]}) + + # Tests for insert conflict resolution function + + # Using a conflict function + - cd: tbl.insert({'id':42, 'foo':1, 'bar':1}) + ot: partial({'inserted':1}) + - py: tbl.insert({'id':42, 'foo':5, 'bar':5}, conflict=lambda id, old_row, new_row: old_row.merge(new_row.pluck("bar"))) + ot: partial({'replaced':1}) + - py: tbl.get(42) + ot: {'id':42, 'foo':1, 'bar':5} + - rb: tbl.insert({:id=>42, :foo=>6, :bar=>6}, conflict: lambda {|id, old_row, new_row| return old_row.merge(new_row.pluck("bar"))}) + ot: partial({'replaced':1}) + - rb: tbl.get(42) + ot: {'id':42, 'foo':1, 'bar':6} + - js: tbl.insert({'id':42, 'foo':7, 'bar':7}, {conflict: function(id, old_row, new_row) {return old_row.merge(new_row.pluck("bar"))}}) + ot: partial({'replaced':1}) + - js: tbl.get(42) + ot: {'id':42, 'foo':1, 'bar':7} + + # Inserting and deleting an item + - js: tbl.insert({id: "toggle"},{conflict: function(x,y,z) { return null},returnChanges: true}) + ot: partial({'inserted': 1}) + - js: tbl.insert({id: "toggle"},{conflict: function(x,y,z) { return null},returnChanges: true}) + ot: partial({'deleted': 1}) + + # Returning the wrong thing from the conflict function + - py: tbl.insert({'id':42, 'foo':1, 'bar':1}, conflict=lambda a,b,c: 2) + ot: partial({'first_error': 'Inserted value must be an OBJECT (got NUMBER):\n2'}) + + # Incorrect Arity + - py: tbl.insert({'id':42}, conflict=lambda a,b: a) + ot: err("ReqlQueryLogicError", "The conflict function passed to `insert` should expect 3 arguments.") + + # Non atomic operation + - py: tbl.insert({'id':42}, conflict=lambda a,b,c: tbl.get(42)) + ot: err("ReqlQueryLogicError", "The conflict function passed to `insert` must be deterministic.") + + - py: tbl.insert({'id':42}, conflict=lambda a,b,c: {'id':42, 'num':'424'}) + ot: partial({'replaced': 1}) + - py: tbl.get(42) + ot: {'id':42, 'num':'424'} + + # Get unreturnable data + - cd: r.minval + ot: err('ReqlQueryLogicError','Cannot convert `r.minval` to JSON.') + + - cd: r.maxval + ot: err('ReqlQueryLogicError','Cannot convert `r.maxval` to JSON.') + + # clean up + - cd: r.db('test').table_drop('test2') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/mutation/replace.yaml b/ext/librethinkdbxx/test/upstream/mutation/replace.yaml new file mode 100644 index 00000000..38cf4f85 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/replace.yaml @@ -0,0 +1,110 @@ +desc: Tests replacement of selections +table_variable_name: tbl +tests: + + # Set up some data + + - py: tbl.insert([{'id':i} for i in xrange(100)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id:i}); + } + return res; + }()) + rb: tbl.insert((1..100).map{ |i| {:id => i } }) + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100}) + + - cd: tbl.count() + ot: 100 + + # Identity + + - py: tbl.get(12).replace(lambda row:{'id':row['id']}) + js: tbl.get(12).replace(function(row) { return {'id':row('id')}; }) + rb: tbl.get(12).replace{ |row| { :id => row[:id] } } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # Replace single row + + - py: tbl.get(12).replace(lambda row:{'id':row['id'], 'a':row['id']}) + js: tbl.get(12).replace(function(row) { return {'id':row('id'), 'a':row('id')}; }) + rb: tbl.get(12).replace{ |row| { :id => row[:id], :a => row[:id] } } + ot: ({'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - py: tbl.get(13).replace(lambda row:null) + js: tbl.get(13).replace(function(row) { return null; }) + rb: tbl.get(13).replace{ |row| null } + ot: ({'deleted':1,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # Replace selection of table + + - py: tbl.between(10, 20, right_bound='closed').replace(lambda row:{'a':1}) + js: tbl.between(10, 20, {'right_bound':'closed'}).replace(function(row) { return {'a':1}; }) + ot: ({'first_error':'Inserted object must have primary key `id`:\n{\n\t\"a\":\t1\n}','deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':10,'skipped':0.0,'inserted':0.0}) + + - py: tbl.filter(lambda row:(row['id'] >= 10) & (row['id'] < 20)).replace(lambda row:{'id':row['id'], 'a':row['id']}) + js: tbl.filter(function(row) { return row('id').ge(10).and(row('id').lt(20))}).replace(function(row) { return {'id':row('id'), 'a':row('id')}; }) + rb: tbl.filter{ |row| + (row[:id] >= 10).and(row[:id] < 20) + }.replace{ |row| + { :id => row[:id], :a => row[:id] } } + ot: ({'deleted':0.0,'replaced':8,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # trying to change pkey of a document + - cd: tbl.get(1).replace({'id':2,'a':1}) + ot: ({'first_error':"Primary key `id` cannot be changed (`{\n\t\"id\":\t1\n}` -> `{\n\t\"a\":\t1,\n\t\"id\":\t2\n}`).",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':1,'skipped':0.0,'inserted':0.0}) + + + # not passing a pkey in the first place + - cd: tbl.get(1).replace({'a':1}) + ot: ({'first_error':"Inserted object must have primary key `id`:\n{\n\t\"a\":\t1\n}",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':1,'skipped':0.0,'inserted':0.0}) + + # check r.row, static value and otherwise + - py: tbl.get(1).replace({'id':r.row['id'],'a':'b'}) + js: tbl.get(1).replace({'id':r.row('id'),'a':'b'}) + rb: tbl.get(1).replace{ |row| { :id => row[:id], :a => 'b' } } + ot: ({'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - cd: tbl.get(1).replace(r.row.merge({'a':'b'})) + rb: tbl.get(1).replace{ |row| row.merge({'a':'b'}) } + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + # test atomicity constraints + - cd: tbl.get(1).replace(r.row.merge({'c':r.js('5')})) + rb: tbl.get(1).replace{ |row| row.merge({'c':r.js('5')}) } + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - cd: tbl.get(1).replace(r.row.merge({'c':tbl.nth(0)})) + rb: tbl.get(1).replace{ |row| row.merge({'c':tbl.nth(0)}) } + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - py: tbl.get(1).replace(r.row.merge({'c':r.js('5')}), non_atomic=True) + js: tbl.get(1).replace(r.row.merge({'c':r.js('5')}), {'nonAtomic':true}) + rb: tbl.get(1).replace({ :non_atomic => true }){ |row| row.merge({ :c => r.js('5') })} + ot: ({'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - cd: tbl.get(1).replace({}, 'foo') + ot: + cd: err('ReqlCompileError', 'Expected 2 arguments but found 3.') + js: err('ReqlCompileError', 'Expected 1 argument (not including options) but found 2.') + + - cd: tbl.get(1).replace({}, {'foo':'bar'}) + py: tbl.get(1).replace({}, foo='bar') + ot: err('ReqlCompileError', 'Unrecognized optional argument `foo`.') + + # Replace whole table + + - py: tbl.replace(lambda row:null) + js: tbl.replace(function(row) { return null; }) + rb: tbl.replace{ |row| null } + ot: ({'deleted':99,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0}) + + - cd: tbl.get('sdfjk').replace({'id':'sdfjk'})['inserted'] + js: tbl.get('sdfjk').replace({'id':'sdfjk'})('inserted') + ot: 1 + - cd: tbl.get('sdfjki').replace({'id':'sdfjk'})['errors'] + js: tbl.get('sdfjki').replace({'id':'sdfjk'})('errors') + ot: 1 + diff --git a/ext/librethinkdbxx/test/upstream/mutation/sync.yaml b/ext/librethinkdbxx/test/upstream/mutation/sync.yaml new file mode 100644 index 00000000..789b3c47 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/sync.yaml @@ -0,0 +1,52 @@ +desc: Tests syncing tables +tests: + + # Set up our test tables + - cd: r.db('test').table_create('test1') + ot: partial({'tables_created':1}) + - cd: r.db('test').table_create('test1soft') + ot: partial({'tables_created':1}) + - cd: r.db('test').table('test1soft').config().update({'durability':'soft'}) + ot: {'skipped':0, 'deleted':0, 'unchanged':0, 'errors':0, 'replaced':1, 'inserted':0} + - def: tbl = r.db('test').table('test1') + - def: tbl_soft = r.db('test').table('test1soft') + - cd: tbl.index_create('x') + ot: partial({'created':1}) + - cd: tbl.index_wait('x').pluck('index', 'ready') + ot: [{'ready':True, 'index':'x'}] + + # This is the only way one can use sync legally at the moment + - cd: tbl.sync() + ot: {'synced':1} + - cd: tbl_soft.sync() + ot: {'synced':1} + - cd: tbl.sync() + ot: {'synced':1} + runopts: + durability: "soft" + - cd: tbl.sync() + ot: {'synced':1} + runopts: + durability: "hard" + + # This is of type table, but sync should still fail (because it makes little sense) + - cd: tbl.between(1, 2).sync() + ot: + cd: err('ReqlQueryLogicError', 'Expected type TABLE but found TABLE_SLICE:', [1]) + py: err('AttributeError', "'Between' object has no attribute 'sync'") + + # These are not even a table. Sync should fail with a different error message + - cd: r.expr(1).sync() + ot: + cd: err("ReqlQueryLogicError", 'Expected type TABLE but found DATUM:', [1]) + py: err('AttributeError', "'Datum' object has no attribute 'sync'") + - js: tbl.order_by({index:'x'}).sync() + rb: tbl.order_by({:index => 'soft'}).sync() + ot: err("ReqlQueryLogicError", 'Expected type TABLE but found TABLE_SLICE:', [1]) + + # clean up + - cd: r.db('test').table_drop('test1') + ot: partial({'tables_dropped':1}) + - cd: r.db('test').table_drop('test1soft') + ot: partial({'tables_dropped':1}) + diff --git a/ext/librethinkdbxx/test/upstream/mutation/update.yaml b/ext/librethinkdbxx/test/upstream/mutation/update.yaml new file mode 100644 index 00000000..3cde0f34 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/mutation/update.yaml @@ -0,0 +1,170 @@ +desc: Tests updates of selections +table_variable_name: tbl, tbl2 +tests: + + # Set up some data + - py: tbl.insert([{'id':i} for i in xrange(100)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id:i}); + } + return res; + }()) + rb: tbl.insert((0...100).map{ |i| { :id => i } }) + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100}) + + - cd: tbl.count() + ot: 100 + + - py: tbl2.insert([{'id':i, 'foo':{'bar':i}} for i in xrange(100)]) + js: | + tbl2.insert(function(){ + var res = [] + for (var i = 0; i < 100; i++) { + res.push({id:i,foo:{bar:i}}); + } + return res; + }()) + rb: tbl2.insert((0...100).map{ |i| { :id => i, :foo => { :bar => i } } }) + ot: ({'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':100}) + + - cd: tbl2.count() + ot: 100 + + # Identity + - py: tbl.get(12).update(lambda row:row) + js: tbl.get(12).update(function(row) { return row; }) + rb: tbl.get(12).update{ |row| row} + ot: {'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # Soft durability point update + - py: tbl.get(12).update(lambda row:{'a':row['id'] + 1}, durability='soft') + js: tbl.get(12).update(function(row) { return {'a':row('id').add(1)}; }, {durability:'soft'}) + rb: tbl.get(12).update({ :durability => 'soft' }) { |row| { :a => row[:id] + 1 } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl.get(12) + ot: {'id':12, 'a':13} + + # Hard durability point update + - py: tbl.get(12).update(lambda row:{'a':row['id'] + 2}, durability='hard') + js: tbl.get(12).update(function(row) { return {'a':row('id').add(2)}; }, {durability:'hard'}) + rb: tbl.get(12).update({ :durability => 'hard' }) { |row| { :a => row[:id] + 2 } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl.get(12) + ot: {'id':12, 'a':14} + + # Wrong durability point update + - py: tbl.get(12).update(lambda row:{'a':row['id'] + 3}, durability='wrong') + js: tbl.get(12).update(function(row) { return {'a':row('id').add(3)}; }, {durability:'wrong'}) + rb: tbl.get(12).update({ :durability => 'wrong' }) { |row| { :a => row[:id] + 3 } } + ot: err('ReqlQueryLogicError', 'Durability option `wrong` unrecognized (options are "hard" and "soft").', [0]) + + - cd: tbl.get(12) + ot: {'id':12, 'a':14} + + # Point update + - py: tbl.get(12).update(lambda row:{'a':row['id']}) + js: tbl.get(12).update(function(row) { return {'a':row('id')}; }) + rb: tbl.get(12).update{ |row| { :a => row[:id] } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl.get(12) + ot: {'id':12, 'a':12} + + # undo the point update + - cd: tbl.get(12).update({'a':r.literal()}) + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # Update selection of table + + - py: tbl.between(10, 20).update(lambda row:{'a':row['id']}) + js: tbl.between(10, 20).update(function(row) { return {'a':row('id')}; }) + rb: tbl.between(10, 20).update{ |row| { :a => row[:id] } } + ot: {'deleted':0.0,'replaced':10,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - py: tbl.filter(lambda row:(row['id'] >= 10) & (row['id'] < 20)).update(lambda row:{'a':row['id']}) + js: tbl.filter(function(row) { return row('id').ge(10).and(row('id').lt(20))}).update(function(row) { return {'a':row('id')}; }) + rb: tbl.filter{ |row| (row[:id] >= 10).and(row[:id] < 20) }.update{ |row| { :a => row[:id] } } + ot: {'deleted':0.0,'replaced':0.0,'unchanged':10,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - py: tbl.filter(lambda row:(row['id'] >= 10) & (row['id'] < 20)).update(lambda row:{'b':row['id']}) + js: tbl.filter(function(row) { return row('id').ge(10).and(row('id').lt(20))}).update(function(row) { return {'b':row('id')}; }) + rb: tbl.filter{ |row| (row[:id] >= 10).and(row[:id] < 20) }.update{ |row| { :b => row[:id] } } + ot: {'deleted':0.0,'replaced':10,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # now undo that update + - cd: tbl.between(10, 20).update({'a':r.literal()}) + ot: {'deleted':0.0,'replaced':10,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # trying to change pkey of a document + - cd: tbl.get(1).update({'id':2,'d':1}) + ot: {'first_error':"Primary key `id` cannot be changed (`{\n\t\"id\":\t1\n}` -> `{\n\t\"d\":\t1,\n\t\"id\":\t2\n}`).",'deleted':0.0,'replaced':0.0,'unchanged':0.0,'errors':1,'skipped':0.0,'inserted':0.0} + + # check r.row, static value and otherwise + - py: tbl.get(1).update({'id':r.row['id'],'d':'b'}) + js: tbl.get(1).update({'id':r.row('id'),'d':'b'}) + rb: tbl.get(1).update{ |row| { :id => row[:id], :d => 'b' } } + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # I don't we don't need the merge, just testing r.row just in case + - cd: tbl.get(1).update(r.row.merge({'d':'b'})) + rb: tbl.get(1).update{ |row| row.merge({'d':'b'}) } + ot: {'deleted':0.0,'replaced':0.0,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # test atomicity constraints (positive and negative test) + - cd: tbl.get(1).update({'d':r.js('5')}) + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - cd: tbl.get(1).update({'d':tbl.nth(0)}) + ot: err('ReqlQueryLogicError', 'Could not prove argument deterministic. Maybe you want to use the non_atomic flag?', [0]) + + - py: tbl.get(1).update({'d':r.js('5')}, non_atomic=True) + js: tbl.get(1).update({'d':r.js('5')}, {'nonAtomic':true}) + rb: tbl.get(1).update({ :d => r.js('5') }, { :non_atomic => true }) + ot: {'deleted':0.0,'replaced':1,'unchanged':0.0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - js: tbl.get(1).update({}, 'foo') + ot: err('ReqlCompileError', 'Expected 1 argument (not including options) but found 2.') + + - js: tbl.get(1).update({}, {'foo':'bar'}) + ot: err('ReqlCompileError', 'Unrecognized optional argument `foo`.') + + # Update whole table + - py: tbl.update(lambda row:{'a':row['id']}) + js: tbl.update(function(row) { return {'a':row('id')}; }) + rb: tbl.update{ |row| { :a => row['id'] } } + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # undo the update on the whole table + - cd: tbl.update({'a':r.literal()}) + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + # recursive merge + - cd: tbl2.update({'foo':{'bar':2}}) + ot: {'deleted':0.0,'replaced':99,'unchanged':1,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.update({'foo':r.literal({'bar':2})}) + ot: {'deleted':0.0,'replaced':0,'unchanged':100,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - rb: tbl2.update{|row| {'foo':r.literal({'bar':2})}} + ot: {'deleted':0.0,'replaced':0,'unchanged':100,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.order_by('id').nth(0) + ot: {'id':0,'foo':{'bar':2}} + + - cd: tbl2.update({'foo':{'buzz':2}}) + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.order_by('id').nth(0) + ot: {'id':0,'foo':{'bar':2,'buzz':2}} + + - cd: tbl2.update({'foo':r.literal(1)}) + ot: {'deleted':0.0,'replaced':100,'unchanged':0,'errors':0.0,'skipped':0.0,'inserted':0.0} + + - cd: tbl2.order_by('id').nth(0) + ot: {'id':0,'foo':1} + diff --git a/ext/librethinkdbxx/test/upstream/parsePolyglot.py b/ext/librethinkdbxx/test/upstream/parsePolyglot.py new file mode 100755 index 00000000..ce048d1b --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/parsePolyglot.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os, re, sys + +# == globals + +printDebug = False + +try: + unicode +except NameError: + unicode = str + +# == + +class yamlValue(unicode): + linenumber = None + def __new__(cls, value, linenumber=None): + if isinstance(value, unicode): + real = unicode.__new__(cls, value) + else: + real = unicode.__new__(cls, value, "utf-8") + if linenumber is not None: + real.linenumber = int(linenumber) + return real + + def __repr__(self): + real = super(yamlValue, self).__repr__() + return real.lstrip('u') + +def parseYAML(source): + + def debug(message): + if printDebug and message: + message = str(message).rstrip() + if message: + print(message) + sys.stdout.flush() + + commentLineRegex = re.compile('^\s*#') + yamlLineRegex = re.compile('^(?P *)((?P- +)(?P.*)|((?P[\w\.]+)(?P: *))?(?P.*))\s*$') + + def parseYAML_inner(source, indent): + returnItem = None + + for linenumber, line in source: + if line == '': # no newline, so EOF + break + + debug('line %d (%d):%s' % (linenumber, indent, line)) + + if line.strip() == '' or commentLineRegex.match(line): # empty or comment line, ignore + debug('\tempty/comment line') + continue + + # - parse line + + parsedLine = yamlLineRegex.match(line) + if not parsedLine: + raise Exception('Unparseable YAML line %d: %s' % (linenumber, line.rstrip())) + + lineIndent = len(parsedLine.group('indent')) + lineItemMarker = parsedLine.group('itemMarker') + lineKey = parsedLine.group('key') or '' + lineKeyExtra = parsedLine.group('keyExtra') or '' + lineContent = (parsedLine.group('content') or parsedLine.group('itemContent') or '').strip() + + # - handle end-of-sections + if lineIndent < indent: + # we have dropped out of this item, push back the line and return what we have + source.send((linenumber, line)) + debug('\tout one level') + return returnItem + + # - array item + if lineItemMarker: + debug('\tarray item') + # item in an array + if returnItem is None: + debug('\tnew array, indent is %d' % lineIndent) + returnItem = [] + indent = lineIndent + elif not isinstance(returnItem, list): + raise Exception('Bad YAML, got a list item while working on a %s on line %d: %s' % (returnItem.__class__.__name__, linenumber, line.rstrip())) + indentLevel = lineIndent + len(lineItemMarker) + source.send((linenumber, (' ' * (indentLevel) )+ lineContent)) + returnItem += [parseYAML_inner(source=source, indent=indent + 1)] + + # - dict item + elif lineKey: + debug('\tdict item') + if returnItem is None: + debug('\tnew dict, indent is %d' % lineIndent) + # new dict + returnItem = {} + indent = lineIndent + elif not isinstance(returnItem, dict): + raise Exception('Bad YAML, got a dict value while working on a %s on line %d: %s' % (returnItem.__class__.__name__, linenumber, line.rstrip())) + indentLevel = lineIndent + len(lineKey) + len(lineKeyExtra) + source.send((linenumber, (' ' * indentLevel) + lineContent)) + returnItem[lineKey] = parseYAML_inner(source=source, indent=indent + 1) + + # - data - one or more lines of text + else: + debug('\tvalue') + if returnItem is None: + returnItem = yamlValue('', linenumber) + if lineContent.strip() in ('|', '|-', '>'): + continue # yaml multiline marker + elif not isinstance(returnItem, yamlValue): + raise Exception('Bad YAML, got a value while working on a %s on line %d: %s' % (returnItem.__class__.__name__, linenumber, line.rstrip())) + if returnItem: + returnItem = yamlValue(returnItem + "\n" + lineContent, returnItem.linenumber) # str subclasses are not fun + else: + returnItem = yamlValue(lineContent, linenumber) + return returnItem + + def parseYAML_generator(source): + if hasattr(source, 'capitalize'): + if os.path.isfile(source): + source = open(source, 'r') + else: + source = source.splitlines(True) + elif hasattr(source, 'readlines'): + pass # the for loop will already work + + backlines = [] + for linenumber, line in enumerate(source): + backline = None + usedLine = False + while usedLine is False or backlines: + if backlines: + backline = yield backlines.pop() + else: + usedLine = True + backline = yield (linenumber + 1, line) + while backline: # loops returning None for every send() + assert isinstance(backline, tuple) + assert isinstance(backline[0], int) + backlines.append(backline) + backline = yield None + + return parseYAML_inner(parseYAML_generator(source), indent=0) + +if __name__ == '__main__': + import optparse, pprint + + parser = optparse.OptionParser() + parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="print debug information") + (options, args) = parser.parse_args() + printDebug = options.debug + + if len(args) < 1: + parser.error('%s needs files to process' % os.path.basename(__file__)) + + for filePath in args: + if not os.path.isfile(filePath): + sys.exit('target is not an existing file: %s' % os.path.basename(__file__)) + + for filePath in args: + print('=== %s' % filePath) + pprint.pprint(parseYAML(filePath)) diff --git a/ext/librethinkdbxx/test/upstream/polymorphism.yaml b/ext/librethinkdbxx/test/upstream/polymorphism.yaml new file mode 100644 index 00000000..14817289 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/polymorphism.yaml @@ -0,0 +1,33 @@ +desc: Tests that manipulation data in tables +table_variable_name: tbl +tests: + + - def: obj = r.expr({'id':0,'a':0}) + + - py: tbl.insert([{'id':i, 'a':i} for i in xrange(3)]) + js: | + tbl.insert(function(){ + var res = [] + for (var i = 0; i < 3; i++) { + res.push({id:i, 'a':i}); + } + return res; + }()) + rb: tbl.insert((0..2).map{ |i| { :id => i, :a => i } }) + ot: ({'deleted':0,'replaced':0,'unchanged':0,'errors':0,'skipped':0,'inserted':3}) + + # Polymorphism + - cd: + - tbl.merge({'c':1}).nth(0) + - obj.merge({'c':1}) + ot: ({'id':0,'c':1,'a':0}) + + - cd: + - tbl.without('a').nth(0) + - obj.without('a') + ot: ({'id':0}) + + - cd: + - tbl.pluck('a').nth(0) + - obj.pluck('a') + ot: ({'a':0}) diff --git a/ext/librethinkdbxx/test/upstream/random.yaml b/ext/librethinkdbxx/test/upstream/random.yaml new file mode 100644 index 00000000..59e2cde6 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/random.yaml @@ -0,0 +1,180 @@ +desc: Tests randomization functions +tests: + +# Test sample + - cd: r.expr([1,2,3]).sample(3).distinct().count() + ot: 3 + - cd: r.expr([1,2,3]).sample(3).count() + ot: 3 + - cd: r.expr([1,2,3,4,5,6]).sample(3).distinct().count() + ot: 3 + - cd: r.expr([1,2,3]).sample(4).distinct().count() + ot: 3 + - rb: r.expr([[1,2,3], 2]).do{|x| x[0].sample(x[1])}.distinct().count() + ot: 2 + - cd: r.expr([1,2,3]).sample(-1) + ot: err('ReqlQueryLogicError', 'Number of items to sample must be non-negative, got `-1`.', [0]) + - cd: r.expr(1).sample(1) + ot: err('ReqlQueryLogicError', 'Cannot convert NUMBER to SEQUENCE', [0]) + - cd: r.expr({}).sample(1) + ot: err('ReqlQueryLogicError', 'Cannot convert OBJECT to SEQUENCE', [0]) + +# Test r.random with floating-point values + # These expressions should be equivalent + - py: + - r.random().do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(1, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(0, 1, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(1, 0, float=True).do(lambda x:r.and_(x.le(1), x.gt(0))) + - r.random(r.expr(0), 1, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1))) + - r.random(1, r.expr(0), float=True).do(lambda x:r.and_(x.le(1), x.gt(0))) + - r.random(r.expr(1), r.expr(0), float=True).do(lambda x:r.and_(x.le(1), x.gt(0))) + ot: True + + # Single-argument + - py: + - r.random(0.495, float=True).do(lambda x:r.and_(x.ge(0), x.lt(0.495))) + - r.random(-0.495, float=True).do(lambda x:r.and_(x.le(0), x.gt(-0.495))) + - r.random(1823756.24, float=True).do(lambda x:r.and_(x.ge(0), x.lt(1823756.24))) + - r.random(-1823756.24, float=True).do(lambda x:r.and_(x.le(0), x.gt(-1823756.24))) + ot: True + + # Non-zero-based random numbers + - py: + - r.random(10.5, 20.153, float=True).do(lambda x:r.and_(x.ge(10.5), x.lt(20.153))) + - r.random(20.153, 10.5, float=True).do(lambda x:r.and_(x.le(20.153), x.gt(10.5))) + - r.random(31415926.1, 31415926, float=True).do(lambda x:r.and_(x.le(31415926.1), x.gt(31415926))) + ot: True + + # Negative random numbers + - py: + - r.random(-10.5, 20.153, float=True).do(lambda x:r.and_(x.ge(-10.5), x.lt(20.153))) + - r.random(-20.153, -10.5, float=True).do(lambda x:r.and_(x.ge(-20.153), x.lt(-10.5))) + - r.random(-31415926, -31415926.1, float=True).do(lambda x:r.and_(x.le(-31415926), x.gt(-31415926.1))) + ot: True + + # There is a very small chance of collision here + - py: + - r.expr([r.random(), r.random()]).distinct().count() + - r.expr([r.random(1, float=True), r.random(1, float=True)]).distinct().count() + - r.expr([r.random(0, 1, float=True), r.random(0, 1, float=True)]).distinct().count() + ot: 2 + + # Zero range random + - py: + - r.random(0, float=True).eq(0) + - r.random(5, 5, float=True).eq(5) + - r.random(-499384756758, -499384756758, float=True).eq(-499384756758) + - r.random(-93.94757, -93.94757, float=True).eq(-93.94757) + - r.random(294.69148, 294.69148, float=True).eq(294.69148) + ot: True + + # Test limits of doubles + - def: + py: float_max = sys.float_info.max + js: float_max = Number.MAX_VALUE + rb: float_max = Float::MAX + - def: + py: float_min = sys.float_info.min + js: float_min = Number.MIN_VALUE + rb: float_min = Float::MIN + - py: + - r.random(-float_max, float_max, float=True).do(lambda x:r.and_(x.ge(-float_max), x.lt(float_max))) + - r.random(float_max, -float_max, float=True).do(lambda x:r.and_(x.le(float_max), x.gt(-float_max))) + - r.random(float_min, float_max, float=True).do(lambda x:r.and_(x.ge(float_min), x.lt(float_max))) + - r.random(float_min, -float_max, float=True).do(lambda x:r.and_(x.le(float_min), x.gt(-float_max))) + - r.random(-float_min, float_max, float=True).do(lambda x:r.and_(x.ge(-float_min), x.lt(float_max))) + - r.random(-float_min, -float_max, float=True).do(lambda x:r.and_(x.le(-float_min), x.gt(-float_max))) + ot: True + +# Test r.random with integer values + - def: + py: upper_limit = 2**53 - 1 + js: upper_limit = Math.pow(2,53) - 1 + rb: upper_limit = 2**53 - 1 + - def: + py: lower_limit = 1 - (2**53) + js: lower_limit = 1 - Math.pow(2,53) + rb: lower_limit = 1 - (2**53) + # These expressions should be equivalent + - py: + - r.random(256).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(0, 256).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(r.expr(256)).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(r.expr(0), 256).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(0, r.expr(256)).do(lambda x:r.and_(x.ge(0), x.lt(256))) + - r.random(r.expr(0), r.expr(256)).do(lambda x:r.and_(x.ge(0), x.lt(256))) + ot: True + + # Non-zero-based random numbers + - py: + - r.random(10, 20).do(lambda x:r.and_(x.ge(10), x.lt(20))) + - r.random(9347849, 120937493).do(lambda x:r.and_(x.ge(9347849), x.lt(120937493))) + js: + - r.random(10, 20).do(function(x){return r.and(x.ge(10), x.lt(20))}) + - r.random(9347849, 120937493).do(function(x){return r.and(x.ge(9347849), x.lt(120937493))}) + rb: + - r.random(10, 20).do{|x| r.and(x.ge(10), x.lt(20))} + - r.random(9347849, 120937493).do{|x| r.and(x.ge(9347849), x.lt(120937493))} + ot: True + + # Negative random numbers + - py: + - r.random(-10, 20).do(lambda x:r.and_(x.ge(-10), x.lt(20))) + - r.random(-20, -10).do(lambda x:r.and_(x.ge(-20), x.lt(-10))) + - r.random(-120937493, -9347849).do(lambda x:r.and_(x.ge(-120937493), x.lt(-9347849))) + js: + - r.random(-10, 20).do(function(x){return r.and(x.ge(-10), x.lt(20))}) + - r.random(-20, -10).do(function(x){return r.and(x.ge(-20), x.lt(-10))}) + - r.random(-120937493, -9347849).do(function(x){return r.and(x.ge(-120937493), x.lt(-9347849))}) + rb: + - r.random(-10, 20).do{|x| r.and(x.ge(-10), x.lt(20))} + - r.random(-20, -10).do{|x| r.and(x.ge(-20), x.lt(-10))} + - r.random(-120937493, -9347849).do{|x| r.and(x.ge(-120937493), x.lt(-9347849))} + ot: True + + # There is a very small chance of collision here + - cd: r.expr([r.random(upper_limit), r.random(upper_limit)]).distinct().count() + ot: 2 + - py: r.expr([upper_limit,upper_limit]).map(lambda x:r.random(x)).distinct().count() + js: r.expr([upper_limit,upper_limit]).map(function(x){return r.random(x)}).distinct().count() + rb: r.expr([upper_limit,upper_limit]).map{|x| r.random(x)}.distinct().count() + ot: 2 + + # Error cases + + # Non-integer limits + - cd: r.random(-0.5) + ot: err("ReqlQueryLogicError", "Upper bound (-0.5) could not be safely converted to an integer.", []) + - cd: r.random(0.25) + ot: err("ReqlQueryLogicError", "Upper bound (0.25) could not be safely converted to an integer.", []) + - cd: r.random(-10, 0.75) + ot: err("ReqlQueryLogicError", "Upper bound (0.75) could not be safely converted to an integer.", []) + - cd: r.random(-120549.25, 39458) + ot: err("ReqlQueryLogicError", "Lower bound (-120549.25) could not be safely converted to an integer.", []) + - cd: r.random(-6.5, 8.125) + ot: err("ReqlQueryLogicError", "Lower bound (-6.5) could not be safely converted to an integer.", []) + + # Forced integer random with no bounds + - py: r.random(float=False) + js: r.random({float:false}) + rb: r.random({:float => false}) + ot: err("ReqlQueryLogicError", "Generating a random integer requires one or two bounds.", []) + + # Lower bound not less than upper bound + - cd: r.random(0) + ot: err("ReqlQueryLogicError", "Lower bound (0) is not less than upper bound (0).", []) + - cd: r.random(0, 0) + ot: err("ReqlQueryLogicError", "Lower bound (0) is not less than upper bound (0).", []) + - cd: r.random(515, 515) + ot: err("ReqlQueryLogicError", "Lower bound (515) is not less than upper bound (515).", []) + - cd: r.random(-956, -956) + ot: err("ReqlQueryLogicError", "Lower bound (-956) is not less than upper bound (-956).", []) + - cd: r.random(-10) + ot: err("ReqlQueryLogicError", "Lower bound (0) is not less than upper bound (-10).", []) + - cd: r.random(20, 2) + ot: err("ReqlQueryLogicError", "Lower bound (20) is not less than upper bound (2).", []) + - cd: r.random(2, -20) + ot: err("ReqlQueryLogicError", "Lower bound (2) is not less than upper bound (-20).", []) + - cd: r.random(1456, 0) + ot: err("ReqlQueryLogicError", "Lower bound (1456) is not less than upper bound (0).", []) diff --git a/ext/librethinkdbxx/test/upstream/range.yaml b/ext/librethinkdbxx/test/upstream/range.yaml new file mode 100644 index 00000000..da9a575f --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/range.yaml @@ -0,0 +1,53 @@ +desc: Tests RQL range generation +tests: + - cd: r.range().type_of() + ot: 'STREAM' + + - cd: r.range().limit(4) + ot: [0, 1, 2, 3] + + - cd: r.range(4) + ot: [0, 1, 2, 3] + + - cd: r.range(2, 5) + ot: [2, 3, 4] + + - cd: r.range(0) + ot: [] + + - cd: r.range(5, 2) + ot: [] + + - cd: r.range(-5, -2) + ot: [-5, -4, -3] + + - cd: r.range(-5, 2) + ot: [-5, -4, -3, -2, -1, 0, 1] + + - cd: r.range(2, 5, 8) + ot: err("ReqlCompileError", "Expected between 0 and 2 arguments but found 3.", []) + + - cd: r.range("foo") + ot: err("ReqlQueryLogicError", "Expected type NUMBER but found STRING.", []) + + # Using 9007199254740994 instead of 9007199254740993 due to #2157 + - cd: r.range(9007199254740994) + ot: err_regex("ReqlQueryLogicError", "Number not an integer \\(>2\\^53\\). 9007199254740994", []) + + - cd: r.range(-9007199254740994) + ot: err_regex("ReqlQueryLogicError", "Number not an integer \\(<-2\\^53\\). -9007199254740994", []) + + - cd: r.range(0.5) + ot: err_regex("ReqlQueryLogicError", "Number not an integer. 0\\.5", []) + + - cd: r.range().count() + ot: err("ReqlQueryLogicError", "Cannot use an infinite stream with an aggregation function (`reduce`, `count`, etc.) or coerce it to an array.", []) + + - cd: r.range().coerce_to("ARRAY") + ot: err("ReqlQueryLogicError", "Cannot use an infinite stream with an aggregation function (`reduce`, `count`, etc.) or coerce it to an array.", []) + + - cd: r.range().coerce_to("OBJECT") + ot: err("ReqlQueryLogicError", "Cannot use an infinite stream with an aggregation function (`reduce`, `count`, etc.) or coerce it to an array.", []) + + - cd: r.range(4).count() + ot: 4 diff --git a/ext/librethinkdbxx/test/upstream/regression/1001.yaml b/ext/librethinkdbxx/test/upstream/regression/1001.yaml new file mode 100644 index 00000000..15327350 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1001.yaml @@ -0,0 +1,20 @@ +desc: 1001 (null + between + sindexes) +table_variable_name: tbl +tests: + - cd: tbl.insert({'a':null}) + rb: tbl.insert({:a => null}) + - cd: tbl.index_create('a') + - cd: tbl.index_create('b') + - cd: tbl.index_wait().pluck('index', 'ready') + + - cd: tbl.between(r.minval, r.maxval).count() + ot: 1 + - py: tbl.between(r.minval, r.maxval, index='a').count() + js: tbl.between(r.minval, r.maxval, {index:'a'}).count() + rb: tbl.between(r.minval, r.maxval, :index => 'a').count() + ot: 0 + - py: tbl.between(r.minval, r.maxval, index='b').count() + js: tbl.between(r.minval, r.maxval, {index:'b'}).count() + rb: tbl.between(r.minval, r.maxval, :index => 'b').count() + ot: 0 + diff --git a/ext/librethinkdbxx/test/upstream/regression/1005.yaml b/ext/librethinkdbxx/test/upstream/regression/1005.yaml new file mode 100644 index 00000000..09ab329c --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1005.yaml @@ -0,0 +1,19 @@ +desc: Regression test for issue #1005. +tests: + - py: r.expr(str(r.table_list())) + ot: "r.table_list()" + + - py: r.expr(str(r.table_create('a'))) + ot: "r.table_create('a')" + + - py: r.expr(str(r.table_drop('a'))) + ot: "r.table_drop('a')" + + - py: r.expr(str(r.db('a').table_list())) + ot: "r.db('a').table_list()" + + - py: r.expr(str(r.db('a').table_create('a'))) + ot: "r.db('a').table_create('a')" + + - py: r.expr(str(r.db('a').table_drop('a'))) + ot: "r.db('a').table_drop('a')" diff --git a/ext/librethinkdbxx/test/upstream/regression/1023.yaml b/ext/librethinkdbxx/test/upstream/regression/1023.yaml new file mode 100644 index 00000000..9c017e04 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1023.yaml @@ -0,0 +1,65 @@ +desc: Tests key sorting of all usable types in primary indexes +table_variable_name: tbl +tests: + + + # Test key sorting + - def: + py: binary_a = r.binary(b'') + rb: binary_a = r.binary('') + js: binary_a = Buffer('') + + - def: + py: binary_b = r.binary(b'5aurhbviunr') + rb: binary_b = r.binary('5aurhbviunr') + js: binary_b = Buffer('5aurhbviunr') + + # Define a set of rows in order of increasing sindex keys + - def: + cd: trows = [{'num':0,'id':[0]}, + {'num':1,'id':[1, 2, 3, 4, 0]}, + {'num':2,'id':[1, 2, 3, 4, 4]}, + {'num':3,'id':[1, 2, 3, 4, 4, 5]}, + {'num':4,'id':[1, 2, 3, 4, 8, 1]}, + {'num':5,'id':[1, 3, r.epoch_time(0)]}, + {'num':6,'id':[1, 3, r.epoch_time(0), r.epoch_time(0)]}, + {'num':7,'id':[1, 3, r.epoch_time(0), r.epoch_time(1)]}, + {'num':8,'id':[1, 4, 3, 4, 8, 2]}, + {'num':9,'id':False}, + {'num':10,'id':True}, + {'num':11,'id':-500}, + {'num':12,'id':500}, + {'num':13,'id':binary_a}, + {'num':14,'id':binary_b}, + {'num':15,'id':r.epoch_time(0)}, + {'num':16,'id':''}, + {'num':17,'id':' str'}] + + - def: + cd: expected = r.range(tbl.count()).coerce_to('array') + + - cd: tbl.insert(trows)['inserted'] + js: tbl.insert(trows)('inserted') + ot: 18 + + - rb: tbl.order_by({:index => 'id'}).map{|row| row['num']}.coerce_to('array').eq(expected) + js: tbl.order_by({index:'id'}).map(r.row('num')).coerce_to('array').eq(expected) + py: tbl.order_by(index='id').map(r.row['num']).coerce_to('array').eq(expected) + ot: true + + # Test minval and maxval + - rb: tbl.order_by(:index => 'id').between(r.minval, r.maxval).map{|x| x['num']}.coerce_to('array').eq(expected) + js: tbl.order_by({index:'id'}).between(r.minval, r.maxval).map(r.row('num')).coerce_to('array').eq(expected) + py: tbl.order_by(index='id').between(r.minval, r.maxval).map(r.row['num']).coerce_to('array').eq(expected) + ot: true + + - py: tbl.order_by(index='id').between([1,2,3,4,4],[1,2,3,5]).map(r.row['num']).coerce_to('array') + js: tbl.order_by({index:'id'}).between([1,2,3,4,4],[1,2,3,5]).map(r.row('num')).coerce_to('array') + rb: tbl.order_by(:index => 'id').between([1,2,3,4,4],[1,2,3,5]).map{|x| x['num']}.coerce_to('array') + ot: [2,3,4] + + - py: tbl.order_by(index='id').between([1,2,3,4,4,r.minval],[1,2,3,4,4,r.maxval]).map(r.row['num']).coerce_to('array') + js: tbl.order_by({index:'id'}).between([1,2,3,4,4,r.minval],[1,2,3,4,4,r.maxval]).map(r.row('num')).coerce_to('array') + rb: tbl.order_by(:index => 'id').between([1,2,3,4,4,r.minval],[1,2,3,4,4,r.maxval]).map{|x| x['num']}.coerce_to('array') + ot: [3] + diff --git a/ext/librethinkdbxx/test/upstream/regression/1081.yaml b/ext/librethinkdbxx/test/upstream/regression/1081.yaml new file mode 100644 index 00000000..ef7d2fc4 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1081.yaml @@ -0,0 +1,39 @@ +desc: 1081 union two streams +tests: + + - rb: r.db('test').table_create('t1081') + def: t = r.db('test').table('t1081') + + - rb: t.insert([{'id':0}, {'id':1}]) + + - rb: r([]).union([]).typeof + ot: ("ARRAY") + - rb: t.union(t).typeof + ot: ("STREAM") + - rb: t.union([]).typeof + ot: ("STREAM") + + - rb: r.db('test').table_drop('t1081') + + - rb: r.table_create('1081') + ot: partial({'tables_created':1}) + + - rb: r.table('1081').insert({:password => 0})[:inserted] + ot: 1 + + - rb: r.table('1081').index_create('password') + ot: ({'created':1}) + - rb: r.table('1081').index_wait('password').pluck('index', 'ready') + ot: ([{'ready':True, 'index':'password'}]) + + - rb: r.table('1081').get_all(0, :index => 'password').typeof + ot: ("SELECTION") + - rb: r.table('1081').get_all(0, :index => 'password').without('id').typeof + ot: ("STREAM") + - rb: r.table('1081').get_all(0, 0, :index => 'password').typeof + ot: ("SELECTION") + - rb: r.table('1081').get_all(0, 0, :index => 'password').without('id').typeof + ot: ("STREAM") + + - rb: r.table_drop('1081') + ot: partial({'tables_dropped':1}) diff --git a/ext/librethinkdbxx/test/upstream/regression/1132.yaml b/ext/librethinkdbxx/test/upstream/regression/1132.yaml new file mode 100644 index 00000000..6208b5d6 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1132.yaml @@ -0,0 +1,4 @@ +desc: 1132 JSON duplicate key +tests: + - cd: r.json('{"a":1,"a":2}') + ot: err("ReqlQueryLogicError", "Duplicate key \"a\" in JSON.", []) diff --git a/ext/librethinkdbxx/test/upstream/regression/1133.yaml b/ext/librethinkdbxx/test/upstream/regression/1133.yaml new file mode 100644 index 00000000..853a3566 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1133.yaml @@ -0,0 +1,19 @@ +desc: Regression tests for issue #1133, which concerns circular references in the drivers. + +tests: + - def: a = {} + - def: b = {'a':a} + - def: a['b'] = b + + - cd: r.expr(a) + ot: + cd: err('ReqlDriverCompileError', 'Nesting depth limit exceeded.', []) + rb: err('ReqlDriverCompileError', 'Maximum expression depth exceeded (you can override this with `r.expr(X, MAX_DEPTH)`).', []) + + - cd: r.expr({'a':{'a':{'a':{'a':{'a':{'a':{'a':{}}}}}}}}, 7) + ot: + cd: err('ReqlDriverCompileError', 'Nesting depth limit exceeded.', []) + rb: err('ReqlDriverCompileError', 'Maximum expression depth exceeded (you can override this with `r.expr(X, MAX_DEPTH)`).', []) + + - cd: r.expr({'a':{'a':{'a':{'a':{'a':{'a':{'a':{}}}}}}}}, 10) + ot: ({'a':{'a':{'a':{'a':{'a':{'a':{'a':{}}}}}}}}) diff --git a/ext/librethinkdbxx/test/upstream/regression/1155.yaml b/ext/librethinkdbxx/test/upstream/regression/1155.yaml new file mode 100644 index 00000000..08dbb6c7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1155.yaml @@ -0,0 +1,5 @@ +desc: 1155 -- Empty batched_replaces_t constructed +table_variable_name: tbl +tests: + - rb: tbl.insert([{:id => '2'}, {:id => '4'}])['inserted'] + ot: 2 diff --git a/ext/librethinkdbxx/test/upstream/regression/1179.yaml b/ext/librethinkdbxx/test/upstream/regression/1179.yaml new file mode 100644 index 00000000..1c04c051 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1179.yaml @@ -0,0 +1,26 @@ +desc: 1179 -- BRACKET term +table_variable_name: tbl +tests: + - js: r.expr([1])(r.expr(0)) + py: r.expr([1])[r.expr(0)] + rb: r.expr([1])[r.expr(0)] + ot: 1 + - js: r.expr({"foo":1})('foo') + ot: 1 + - js: r.expr([1])(0) + ot: 1 + - js: tbl.insert([{'id':42},{'id':4},{'id':89},{'id':6},{'id':43}]).pluck('inserted','first_error') + ot: ({'inserted':5}) + + # test [] grouped data semantics + - js: tbl.group('id')(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + - js: tbl.coerce_to('array').group('id')(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + + # test nth grouped data semantics + - js: tbl.group('id').nth(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + - js: tbl.coerce_to('array').group('id').nth(0) + ot: ([{"group":4,"reduction":{"id":4}},{"group":6,"reduction":{"id":6}},{"group":42,"reduction":{"id":42}},{"group":43,"reduction":{"id":43}},{"group":89,"reduction":{"id":89}}] ) + diff --git a/ext/librethinkdbxx/test/upstream/regression/1468.yaml b/ext/librethinkdbxx/test/upstream/regression/1468.yaml new file mode 100644 index 00000000..0b8cfc15 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1468.yaml @@ -0,0 +1,7 @@ +desc: 1468 -- Empty batched_replaces_t constructed +table_variable_name: tbl +tests: + - rb: tbl.insert([{}, {}, {}])['inserted'] + ot: (3) + - rb: tbl.replace(non_atomic:'true'){|row| r.js("{}")} + ot: ({"unchanged"=>0,"skipped"=>0,"replaced"=>0,"inserted"=>0,"first_error"=>"Cannot convert javascript `undefined` to ql::datum_t.","errors"=>3,"deleted"=>0}) diff --git a/ext/librethinkdbxx/test/upstream/regression/1789.yaml b/ext/librethinkdbxx/test/upstream/regression/1789.yaml new file mode 100644 index 00000000..5ad614b7 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/1789.yaml @@ -0,0 +1,22 @@ +desc: 1789 -- deleting a secondary index on a table that contains non-inline stored documents corrupts db +table_variable_name: tbl +tests: + - rb: tbl.insert({:foo => 'a', :data => "AAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}).pluck('inserted') + ot: ({'inserted':1}) + + - rb: tbl.index_create('foo') + ot: ({'created':1}) + + - rb: tbl.index_wait('foo').pluck('index', 'ready') + ot: ([{'index':'foo', 'ready':true}]) + + - rb: tbl.index_drop('foo') + ot: ({'dropped':1}) + + - rb: tbl.coerce_to('ARRAY').count() + ot: (1) + diff --git a/ext/librethinkdbxx/test/upstream/regression/2052.yaml b/ext/librethinkdbxx/test/upstream/regression/2052.yaml new file mode 100644 index 00000000..c7c62bc2 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2052.yaml @@ -0,0 +1,10 @@ +desc: 2052 -- Verify that the server rejects bogus global options. +tests: + - cd: r.expr(1) + runopts: + array_limit: 16 + ot: 1 + - cd: r.expr(1) + runopts: + obviously_bogus: 16 + ot: err("ReqlCompileError", "Unrecognized global optional argument `obviously_bogus`.", []) diff --git a/ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml b/ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml new file mode 100644 index 00000000..6f646bac --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2399.rb.yaml @@ -0,0 +1,45 @@ +desc: 2399 literal terms not removed under certain circumstances +table_variable_name: t +tests: + - rb: t.insert({}) + - rb: t.update({:a => {:b => r.literal({})}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{}}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => r.literal()}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a': {}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => {:c => {:d => r.literal({})}}}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{'c':{'d':{}}}}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => [[[{:c => r.literal({})}]]]}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':[[[{'c':{}}]]]}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => [r.literal()]}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':[]}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => {:a => 'A', :b => 'B', :c => 'C', :cc => r.literal(), :d => 'D'}}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{'a':'A', 'b':'B', 'c':'C', 'd':'D'}}}] + - rb: t.delete() + + - rb: t.insert({}) + - rb: t.update({:a => {:b => {:a => 'A', :b => 'B', :c => 'C', :cc => r.literal('CC'), :d => 'D'}}}) + - rb: t.without('id').coerce_to("ARRAY") + ot: [{'a':{'b':{'a':'A', 'b':'B', 'c':'C', 'cc':'CC', 'd':'D'}}}] + - rb: t.delete() + diff --git a/ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml b/ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml new file mode 100644 index 00000000..70f91761 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2639.rb.yaml @@ -0,0 +1,8 @@ +desc: 2639 -- Coroutine stacks should not overflow during the query compilation phase. +tests: + - rb: r.expr({id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:{id:1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}, 1000) + ot: partial({}) + + - rb: r.expr([[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],1000).and(nil) + ot: nil + diff --git a/ext/librethinkdbxx/test/upstream/regression/2696.yaml b/ext/librethinkdbxx/test/upstream/regression/2696.yaml new file mode 100644 index 00000000..3ca88b55 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2696.yaml @@ -0,0 +1,6 @@ +desc: Regression test for issue 2696, delete_at with end bounds. +tests: + - cd: r.expr([1,2,3,4]).delete_at(4,4) + ot: [1,2,3,4] + - cd: r.expr([]).delete_at(0,0) + ot: [] diff --git a/ext/librethinkdbxx/test/upstream/regression/2697.yaml b/ext/librethinkdbxx/test/upstream/regression/2697.yaml new file mode 100644 index 00000000..ba488172 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2697.yaml @@ -0,0 +1,31 @@ +desc: 2697 -- Array insert and splice operations don't check array size limit. +table_variable_name: tbl +tests: + # make enormous > 100,000 element array + - def: ten_l = r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + - js: tbl.insert({'id':1, 'a':r.expr(ten_l).concatMap(function(l) { return ten_l }).concatMap(function(l) { return ten_l }).concatMap(function(l) { return ten_l }).concatMap(function(l) { return ten_l })}).pluck('first_error', 'inserted') + py: tbl.insert({'id':1, 'a':r.expr(ten_l).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11)))}).pluck('first_error', 'inserted') + rb: tbl.insert({'id':1, 'a':r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}}).pluck('first_error', 'inserted') + ot: ({'inserted':1}) + - cd: tbl.get(1).replace({'id':1, 'a':r.row['a'].splice_at(0, [2])}).pluck('first_error') + js: tbl.get(1).replace({'id':1, 'a':r.row('a').spliceAt(0, [2])}).pluck('first__error') + rb: tbl.get(1).replace{|old| {:id => 1, :a => old['a'].splice_at(0, [2])}}.pluck('first_error') + ot: ({'first_error':'Array over size limit `100000`.'}) + - cd: tbl.get(1)['a'].count() + js: tbl.get(1)('a').count() + ot: 100000 + - cd: tbl.get(1).replace({'id':1, 'a':r.row['a'].insert_at(0, [2])}).pluck('first_error') + js: tbl.get(1).replace({'id':1, 'a':r.row('a').insertAt(0, [2])}).pluck('first__error') + rb: tbl.get(1).replace{|old| {:id => 1, :a => old['a'].insert_at(0, [2])}}.pluck('first_error') + ot: ({'first_error':'Array over size limit `100000`.'}) + - cd: tbl.get(1)['a'].count() + js: tbl.get(1)('a').count() + ot: 100000 + - js: r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).spliceAt(0, [1]).count() + py: r.expr(ten_l).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).splice_at(0, [1]).count() + rb: r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.splice_at(0, [1]).count() + ot: err("ReqlResourceLimitError", "Array over size limit `100000`.", []) + - js: r.expr([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).concatMap(function(l) { return [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }).insertAt(0, [1]).count() + py: r.expr(ten_l).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).concat_map(lambda l:list(range(1,11))).insert_at(0, [1]).count() + rb: r.expr(ten_l).concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.concat_map {|l| ten_l}.insert_at(0, [1]).count() + ot: err("ReqlResourceLimitError", "Array over size limit `100000`.", []) diff --git a/ext/librethinkdbxx/test/upstream/regression/2709.yaml b/ext/librethinkdbxx/test/upstream/regression/2709.yaml new file mode 100644 index 00000000..a5ffe12d --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2709.yaml @@ -0,0 +1,21 @@ +desc: 2709 -- Guarantee failed with [max_els >= min_els] +table_variable_name: tbl +tests: + - py: tbl.insert([{'result':i} for i in range(1,1000)]).pluck('first_error', 'inserted') + runopts: + min_batch_rows: 10 + max_batch_rows: 13 + ot: ({'inserted':999}) + + - py: tbl.map(lambda thing:'key').count() + runopts: + min_batch_rows: 10 + max_batch_rows: 13 + ot: (999) + + - py: tbl.map(lambda thing:'key').count() + runopts: + min_batch_rows: 10 + max_batch_rows: 13 + ot: (999) + diff --git a/ext/librethinkdbxx/test/upstream/regression/2710.yaml b/ext/librethinkdbxx/test/upstream/regression/2710.yaml new file mode 100644 index 00000000..ac5fa738 --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2710.yaml @@ -0,0 +1,6 @@ +desc: Test pseudo literal strings in JSON. +tests: + - js: r.expr({"a":{"b":1, "c":2}}).merge(r.json('{"a":{"$reql_'+'type$":"LITERAL", "value":{"b":2}}}')) + py: r.expr({"a":{"b":1, "c":2}}).merge(r.json('{"a":{"$reql_type$":"LITERAL", "value":{"b":2}}}')) + rb: r.expr({:a => {:b => 1, :c => 2}}).merge(r.json('{"a":{"$reql_type$":"LITERAL", "value":{"b":2}}}')) + ot: ({'a':{'b':2}}) diff --git a/ext/librethinkdbxx/test/upstream/regression/2766.yaml b/ext/librethinkdbxx/test/upstream/regression/2766.yaml new file mode 100644 index 00000000..ffb562fb --- /dev/null +++ b/ext/librethinkdbxx/test/upstream/regression/2766.yaml @@ -0,0 +1,25 @@ +desc: Stop people treating ptypes as objects +tests: + - cd: r.now()['epoch_time'] + js: r.now()('epoch_time') + ot: err("ReqlQueryLogicError", "Cannot call `bracket` on objects of type `PTYPE
FieldTypeDescriptionWritable
addressstring10-digit hex ZeroTier addressno
lastUnicastFrameintegerTime of last unicast frame in ms since epochno
lastMulticastFrameintegerTime of last multicast frame in ms since epochno
versionMajorintegerMajor version of remote if knownno
versionMinorintegerMinor version of remote if knownno
versionRevintegerRevision of remote if knownno